ESP32-CAM transmits video to the public network server and forwards the video data stream

        Most of the current articles about esp32-cam are about how to transmit video on the intranet (under the same network environment). Even on the transmission server, the video image is only displayed on the server, and the video data stream cannot be forwarded to the client. terminal. Let me first introduce the general idea. It is very simple. esp32-cam sends the data stream to the public network server, and the server transmits the data to the user's computer as it is. esp32-cam uses the C language, and the public network server uses nodejs. The computer client uses python                                           esp32-cam ==> Public network server ==> Computer client

All transmissions are carried out under the tcp protocol, so friends remember to select the tcp protocol when opening a port on the public server. First of all, let me state that the codes of esp32-cam and computer client are based on the code of the blogger dsxcode . , the following two parts of the code are only reproduced. Reference link: ESP32 CAM and server (python) TCP video transmission_dsxcode's blog-CSDN blog_esp32 tcp data transmission

The above code is written in Arduino c language using esp32-cam:

#include <Arduino.h>
#include <WiFi.h>
#include "esp_camera.h"
#include <vector>
 
const char *ssid = "xxxx"; //wifi用户名
const char *password = "xxxx"; //wifi密码
const IPAddress serverIP(xxxx);  //你自己的公网服务器ip地址
uint16_t serverPort = xxxx;         //服务器端口号(tcp协议)
 
#define maxcache 1430
 
WiFiClient client; //声明一个客户端对象，用于与服务器进行连接
 
//CAMERA_MODEL_AI_THINKER类型摄像头的引脚定义
#define PWDN_GPIO_NUM     32
#define RESET_GPIO_NUM    -1
#define XCLK_GPIO_NUM      0
#define SIOD_GPIO_NUM     26
#define SIOC_GPIO_NUM     27
 
#define Y9_GPIO_NUM       35
#define Y8_GPIO_NUM       34
#define Y7_GPIO_NUM       39
#define Y6_GPIO_NUM       36
#define Y5_GPIO_NUM       21
#define Y4_GPIO_NUM       19
#define Y3_GPIO_NUM       18
#define Y2_GPIO_NUM        5
#define VSYNC_GPIO_NUM    25
#define HREF_GPIO_NUM     23
#define PCLK_GPIO_NUM     22
 
static camera_config_t camera_config = {
    .pin_pwdn = PWDN_GPIO_NUM,
    .pin_reset = RESET_GPIO_NUM,
    .pin_xclk = XCLK_GPIO_NUM,
    .pin_sscb_sda = SIOD_GPIO_NUM,
    .pin_sscb_scl = SIOC_GPIO_NUM,
    
    .pin_d7 = Y9_GPIO_NUM,
    .pin_d6 = Y8_GPIO_NUM,
    .pin_d5 = Y7_GPIO_NUM,
    .pin_d4 = Y6_GPIO_NUM,
    .pin_d3 = Y5_GPIO_NUM,
    .pin_d2 = Y4_GPIO_NUM,
    .pin_d1 = Y3_GPIO_NUM,
    .pin_d0 = Y2_GPIO_NUM,
    .pin_vsync = VSYNC_GPIO_NUM,
    .pin_href = HREF_GPIO_NUM,
    .pin_pclk = PCLK_GPIO_NUM,
    
    .xclk_freq_hz = 20000000,
    .ledc_timer = LEDC_TIMER_0,
    .ledc_channel = LEDC_CHANNEL_0,
    
    .pixel_format = PIXFORMAT_JPEG,
    .frame_size = FRAMESIZE_VGA,
    .jpeg_quality = 12,
    .fb_count = 1,
};
void wifi_init()
{
    WiFi.mode(WIFI_STA);
    WiFi.setSleep(false); //关闭STA模式下wifi休眠，提高响应速度
    WiFi.begin(ssid, password);
    while (WiFi.status() != WL_CONNECTED)
    {
        delay(500);
        Serial.print(".");
    }
    Serial.println("WiFi Connected!");
    Serial.print("IP Address:");
    Serial.println(WiFi.localIP());
}
esp_err_t camera_init() {
    //initialize the camera
    esp_err_t err = esp_camera_init(&camera_config);
    if (err != ESP_OK) {
        Serial.println("Camera Init Failed");
        return err;
    }
    sensor_t * s = esp_camera_sensor_get();
    //initial sensors are flipped vertically and colors are a bit saturated
    if (s->id.PID == OV2640_PID) {
    //        s->set_vflip(s, 1);//flip it back
    //        s->set_brightness(s, 1);//up the blightness just a bit
    //        s->set_contrast(s, 1);
    }
    Serial.println("Camera Init OK!");
    return ESP_OK;
}
 
void setup()
{
    Serial.begin(115200);
    wifi_init();
    camera_init();
}
 
void loop()
{
    Serial.println("Try To Connect TCP Server!");
    if (client.connect(serverIP, serverPort)) //尝试访问目标地址
    {
        Serial.println("Connect Tcp Server Success!");
        client.println("Frame Begin");  //46 72 61 6D 65 20 42 65 67 69 6E // 0D 0A 代表换行  //向服务器发送数据
        while (1){       
          camera_fb_t * fb = esp_camera_fb_get();
          uint8_t * temp = fb->buf; //这个是为了保存一个地址，在摄像头数据发送完毕后需要返回，否则会出现板子发送一段时间后自动重启，不断重复
          if (!fb)
          {
              Serial.println( "Camera Capture Failed");
          }
          else
          { 
            //先发送Frame Begin 表示开始发送图片 然后将图片数据分包发送 每次发送1430 余数最后发送 
            //完毕后发送结束标志 Frame Over 表示一张图片发送完毕 
            client.print("Frame Begin"); //一张图片的起始标志
            // 将图片数据分段发送
            int leng = fb->len;
            int timess = leng/maxcache;
            int extra = leng%maxcache;
            for(int j = 0;j< timess;j++)
            {
              client.write(fb->buf, maxcache); 
              for(int i =0;i< maxcache;i++)
              {
                fb->buf++;
              }
            }
            client.write(fb->buf, extra);
            client.print("Frame Over");      // 一张图片的结束标志
            Serial.print("This Frame Length:");
            Serial.print(fb->len);
            Serial.println(".Succes To Send Image For TCP!");
            //return the frame buffer back to the driver for reuse
            fb->buf = temp; //将当时保存的指针重新返还
            esp_camera_fb_return(fb);  //这一步在发送完毕后要执行，具体作用还未可知。        
          }
          delay(30);//短暂延时 增加数据传输可靠性
        }
        /*
        while (client.connected() || client.available()) //如果已连接或有收到的未读取的数据
        {
            if (client.available()) //如果有数据可读取
            {
                String line = client.readStringUntil('\n'); //读取数据到换行符
                Serial.print("ReceiveData：");
                Serial.println(line);
                client.print("--From ESP32--:Hello Server!");    
            }
        }
        Serial.println("close connect!");
        client.stop(); //关闭客户端
        */
    }
    else
    {
        Serial.println("Connect To Tcp Server Failed!After 10 Seconds Try Again!");
        client.stop(); //关闭客户端
    }
    delay(10000);
}

Client code, written in Python:

import socket
import threading
import time
import numpy as np
import cv2

begin_data = b'Frame Begin'
end_data = b'Frame Over'


# 接收数据
# ESP32发送一张照片的流程
# 先发送Frame Begin 表示开始发送图片 然后将图片数据分包发送 每次发送1430 余数最后发送
# 完毕后发送结束标志 Frame Over 表示一张图片发送完毕
# 1430 来自ESP32cam发送的一个包大小为1430 接收到数据 data格式为b''
def handle_sock(sock, addr):
    temp_data = b''
    t1 = int(round(time.time() * 1000))
    while True:
        data = sock.recv(1430)
        # 如果这一帧数据包的开头是 b'Frame Begin' 则是一张图片的开始
        if data[0:len(begin_data)] == begin_data:
            # 将这一帧数据包的开始标志信息（b'Frame Begin'）清除   因为他不属于图片数据
            data = data[len(begin_data):len(data)]
            # 判断这一帧数据流是不是最后一个帧 最后一针数据的结尾时b'Frame Over'
            while data[-len(end_data):] != end_data:
                temp_data = temp_data + data  # 不是结束的包 将数据添加进temp_data
                data = sock.recv(1430)  # 继续接受数据 直到接受的数据包包含b'Frame Over' 表示是这张图片的最后一针
            # 判断为最后一个包 将数据去除 结束标志信息 b'Frame Over'
            temp_data = temp_data + data[0:(len(data) - len(end_data))]  # 将多余的（\r\nFrame Over）去掉 其他放入temp_data
            # 显示图片
            receive_data = np.frombuffer(temp_data, dtype='uint8')  # 将获取到的字符流数据转换成1维数组
            r_img = cv2.imdecode(receive_data, cv2.IMREAD_COLOR)  # 将数组解码成图像
            r_img = r_img.reshape(480, 640, 3)
            t2 = int(round(time.time() * 1000))
            fps = 1000 // (t2 - t1)
            cv2.putText(r_img, "FPS" + str(fps), (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
            cv2.imshow('server_frame', r_img)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
            t1 = t2
            # print("接收到的数据包大小：" + str(len(temp_data)))  # 显示该张照片数据大小
            temp_data = b''  # 清空数据 便于下一章照片使用


server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.connect(("xxxx", xxxx))
server.send('monitor'.encode("utf-8"))
# server.bind(('0.0.0.0', 9090))
# server.listen(5)
# CONNECTION_LIST = []

# 主线程循环接收客户端连接
handle_sock(server, '')  # mac电脑直接在主线程运行
# while True:
    # sock, addr = server.accept()
    # CONNECTION_LIST.append(sock)
    # print('Connect--{}'.format(addr))
    # 连接成功后开一个线程用于处理客户端
    # handle_sock(server, '')  # mac电脑直接在主线程运行
    # client_thread = threading.Thread(target=handle_sock, args=(sock, addr))
    # client_thread.start()

Here is a special note. In the last main thread of the python code, since I am running on a Mac computer, I cannot open a new thread, otherwise an error will be reported. If you are running on a Windows computer, please open the comment and create new thread to run.

The following is the server-side code written in nodejs, which is used for transparent transmission and forwarding of video data streams:

// 创建tcp连接服务
const net = require('net')
const { size } = require('underscore')
const HOST = 'xxxx'
const PORT = xxxx

// 创建udp
// const dgram = require("dgram")
// const server = dgram.createSocket("udp4")

// 统计连接客户端的个数
var count = 0

// 创建slave_server服务
const slave_server = new net.createServer()
// slave_server.setEncoding = 'UTF-8'

// 保存监视器的socket
var s

// 获得一个连接，该链接自动关联scoket对象
var tcp_sock = null
slave_server.on('connection', sock => {
  tcp_sock = sock
  sock.name = ++count
  console.log(`当前连接客户端数：${count}`)

  // 接收client发来的信息  
  sock.on('data', data => {
    // console.log(`客户端${sock.name}发来一个信息：${data}`)

    // 判断是否为监视器发来的链接
    if (data == 'monitor') {
      
      // 则把当前的socket保存起来
      s = sock
    } else {
      s.write(data)
    }
  })

  // 为socket添加error事件处理函数
  sock.on('error', error => { //监听客户端异常
    console.log('error' + error)
    sock.end()
  })

  // 服务器关闭时触发，如果存在连接，这个事件不会被触发，直到所有的连接关闭
  sock.on('close', () => {
    console.log(`客户端${sock.name}下线了`)
    count -= 1
  })
})

// listen函数开始监听指定端口
slave_server.listen(PORT, () => {
  console.log(`服务器已启动，运行在：http://xxxx:xxxx`)
})

Let me explain again, first start the server-side code, then run the python-side code on the computer, and finally burn the esp32-cam code. When running for the first time, the computer client may report an error and exit directly. It doesn't matter, just run it again. You can see the video image by using the python code on the computer client. My logic on the server side is that when it receives the 'monitor' string, it thinks it is the client monitor, saves the corresponding socket, and then when the next socket connection is received , the default is esp32-cam. Take out the previously saved socket and forward the video stream data. This method can only have one esp32-cam and one client monitor. Friends who need it can modify it by themselves. code.