Escenario del proyecto:
项目需求,需要做一个基于边缘端的人脸识别远程监控摄像头并在网页前端展示 ,这里采用国产香橙派作为边缘计算终端,安装ubuntu系统,系统中采用v4l2接口对摄像头进行获取,当客户端通过网页进行请求时,服务器通过http服务的形式将一帧帧图像发送给客户端,只要一秒钟能够传送25帧左右,展示效果就是在网页端播放视频:
Descripción del problema 1
Cómo obtener datos de fotogramas de la cámara , aquí conectamos la cámara USB a la placa de desarrollo:
Se puede ver que sí hay una cámara de video0, que es una cámara USB externa
Solución 1:
Use la interfaz V4L2 para mapear los datos leídos desde el estado del kernel al espacio del usuario a través de interrupciones:
El siguiente código asigna el espacio del kernel al espacio del usuario
for(int i = 0; i <4;i++) {
mapbuffer.index = i;
ret = ioctl(fd,VIDIOC_QUERYBUF,&mapbuffer); //从内核空间中查询一个空间作映射
if (ret < 0)
{
perror("查询内核空间失败");
}
//映射到用户空间
mptr[i] = (unsigned char *)mmap(NULL,mapbuffer.length,PROT_READ|PROT_WRITE,MAP_SHARED,fd,mapbuffer.m.offset);
size[i] = mapbuffer.length; //保存映射长度用于后期释放
//查询后通知内核已经放回
ret = ioctl(fd,VIDIOC_QBUF,&mapbuffer);
if (ret < 0)
{
perror("放回失败");
}
}
El siguiente código obtiene datos en el modo kernel a través de interrupciones y copia los datos en la memoria en el modo de usuario, es decir, copia los datos en una matriz, para que otros subprocesos puedan realizar el procesamiento y la visualización de datos.
struct v4l2_buffer readbuffer;
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //每个结构体都需要设置type为这个参赛要记住
ret = ioctl(fd,VIDIOC_DQBUF,&readbuffer);
int read_len = mapbuffer.length;
int send_len = 1024;
//printf("%d \n",read_len);
if (ret < 0)
{
perror("读取数据失败");
}
if(1)
{
void *ptr = mptr[readbuffer.index];
memcpy(jpg_buff, ptr, readbuffer.length);
}
Descripción del problema 2
Para reproducir el video en la parte frontal de la página web, el video primero se compone de cuadros de imágenes, aquí enviamos un cuadro cada intervalo corto, y el cliente y el servidor http adoptan la forma de conexión prolongada de mantenimiento. Para acelerar, aquí uso dos subprocesos, el 1 listo para usar lee el video y coloca el marco de la imagen en una matriz pública, y el subproceso 2 envía los datos en la matriz pública al cliente a través de la transmisión tcp.
Al mismo tiempo, se utiliza una variable de condición para la sincronización entre el subproceso 1 y el subproceso
Solución 2:
Múltiples subprocesos comparten variables globales para acelerar la lectura de videos y el envío de imágenes:
hilo 1
pthread_mutex_lock(&lock);
void *ptr = mptr[readbuffer.index];
memcpy(jpg_buff, ptr, readbuffer.length);//将读取到的图像拷贝到字符数组
pthread_mutex_unlock(&lock);
pthread_cond_signal(&hasNode);
hilo 2
pthread_cond_wait(&hasNode,&lock);
void *ppptr = jpg_buff;
pthread_mutex_unlock(&lock);
El jpg_buff aquí es una matriz global compartida
Descripción del problema 3
Problema de configuración de conexión larga de la página web :
Cuando el servidor devuelva el encabezado del mensaje, agregue
"Content-Type:multipart/x-mixed-replace;
Eso significa que la conexión actual se basa en una conexión larga
solución:
Consejo: Complete la solución específica al problema aquí:
El siguiente es el contenido del encabezado.
/*
HTTP长连接处理客户端请求
"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n"
*/
Esta es la imagen del efecto cuando el programa se está ejecutando:
Debe escribir la URL en el terminal en la misma LAN que la placa de desarrollo: 192.168.0.105:8080
Donde 192.168.0.105 es la dirección IP de la placa de desarrollo
Finalmente adjuntar el código total
#include <stdio.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include <assert.h>
#include <fcntl.h>
#include <jpeglib.h>
#include <linux/fb.h>
#include <linux/videodev2.h>
#include <netinet/in.h>
#include <netinet/ip.h> /* superset of previous */
#include <pthread.h>
#include "sd_usb_pic.h"
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/types.h> /* See NOTES */
#include <unistd.h>
#include <sys/time.h>
#define HTTP_PORT 8080 //HTTP服务器端口号
pthread_mutex_t mutex;
pthread_cond_t hasNode = PTHREAD_COND_INITIALIZER;
pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
char jpg_buff[614400]; //保存从USB摄像头读取的图像矩阵
int send_html_cnt = 0 ;
/*
服务端响应客户端请求
"HTTP/1.1 200 OK\r\n"
"Content-type:image/jpeg\r\n"
"Content-Length:1234\r\n"
"\r\n"
形参:c_fd --客户端套接字
type --文件类型
file --要发送的文件
返回值:0成功,其它失败
*/
void *ppp_tr = NULL;
int fd_fb;
static struct fb_var_screeninfo var; /* LCD可变参数 */
static unsigned int *fb_base = NULL; /* Framebuffer映射基地址 */
int lcd_w = 800 ,lcd_h= 480; //定义显示器分辨率
/*
形参:c_fd --客户端套接字
type --文件类型
file --要发送的文件
返回值:0成功,其它失败
*/
int Http_SendData(int c_fd,const char *type,const char *file)
{
int fd=open(file,O_RDONLY);//打开文件
if(fd<0)return -1;//打开文件失败
struct stat statbuf;
fstat(fd,&statbuf);
if(statbuf.st_size<=0)
{
close(fd);
return -2;
}
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Content-type:%s\r\n"
"Content-Length:%ld\r\n"
"\r\n",type,statbuf.st_size);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
close(fd);
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
while(1)
{
size=read(fd,buff,sizeof(buff));
if(write(c_fd,buff,size)!=size)break;//发送失败
if(size!=sizeof(buff))break;//发送完成
}
close(fd);
return 0;
}
int Http_SendPic(int c_fd,const char *type,const char *file)
{
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Content-type:%s\r\n"
"Content-Length:%ld\r\n"
"\r\n",type,614400);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
int cnt = 0 ;
pthread_mutex_lock(&mutex);
void *ppptr = jpg_buff;
while(1)
{
int size= 1024;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
pthread_mutex_unlock(&mutex);
return 0;
}
int Http_SendPic1(int c_fd,const char *type,const char *file)
{
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Server:LiMeng \r\n"
"Content-type:image/jpeg\r\n"
"Content-Length:%ld\r\n"
"\r\n",614400);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
int cnt = 0 ;
pthread_mutex_lock(&mutex);
void *ppptr = jpg_buff;
while(1)
{
int size= 1024;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
pthread_mutex_unlock(&mutex);
return 0;
}
/*
HTTP长连接处理客户端请求
"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n"
*/
int Http_Content(int c_fd)
{
char buff[1024]={0};
/*建立长连接*/
snprintf(buff,sizeof(buff),"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n");
if(write(c_fd,buff,strlen(buff))!=strlen(buff))return -1;//发送报文头失败
int jpe_image_size = 614400;//保存jpeg图像大小
//int send_html_cnt = 0 ;
clock_t t1;
t1 = clock();
char save_name[10];
int save_idx = 0 ;
struct timeval start_time, end_time;
while(1)
{
//auto beg = clock();
t1 = clock();
pthread_cond_wait(&hasNode,&lock);
//printf("wait time is %d us \n", (clock() - t1) );
auto beg = clock();
gettimeofday(&start_time, NULL);
/*
(1)响应报文头
"Content-type:image/jpeg\r\n"
"Content-Length:666\r\n"
"\r\n"
*/
snprintf(buff,sizeof(buff), "Content-type:image/jpeg\r\n"
"Content-Length:%d\r\n"
"\r\n",jpe_image_size);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -2;//响应报文头失败
}
/*发送jpg图像数据*/
//pthread_mutex_lock(&mutex);//互斥锁上锁
//pthread_cond_wait(&hasNode,&lock);
void *ppptr = jpg_buff;
//void *ppptr = ppp_tr;
//sprintf(save_name,"my_%d.jpg",save_idx++);
//FILE *file=fopen(save_name, "w");
//fwrite(ppptr , 614400, 1,file);
//close(file);
int cnt = 0 ;
while(1)
{
int size= 1024 * 600;
//int size= 1024 ;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
//pthread_mutex_unlock(&mutex);//互斥锁上锁
//pthread_mutex_unlock(&lock);
//sleep(20);
/*
(3)发送间隔字符串
"\r\n"
"--boundarydonotcross\r\n"
*/
strcpy(buff,"\r\n--boundarydonotcross\r\n");
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
break;//发送间隔符失败
}
auto end = clock();
gettimeofday(&end_time, NULL);
double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
//printf("sendtime is %d us \n", (end - beg) );
//printf("send cnt is %d us \n", send_html_cnt );
send_html_cnt++;
//pthread_mutex_unlock(&lock);
//usleep(40000);
//auto beg = clock();
//printf("sendtime is %d ms \n", (end - beg) );
printf("sendtime is %d ms \n", timeuse );
pthread_mutex_unlock(&lock);
usleep(1);
}
return -4;//发送图像数据失败
}
/*线程工作函数*/
void *pth_work(void *arg)
{
int c_fd=*(int *)arg;
free(arg);
char buff[1024]={0};
int size;
size=read(c_fd,buff,sizeof(buff)-1);
if(size<=0)
{
close(c_fd);
pthread_exit(NULL);
}
buff[size]='\0';
printf("buff=%s\n",buff);
if(strstr(buff,"GET / HTTP/1.1"))//请求网页文件
{
Http_SendData(c_fd,"text/html","./html/image.html");
}
else if(strstr(buff,"GET /1.bmp HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/1.bmp");
}
else if(strstr(buff,"GET /my.jpg HTTP/1.1"))
{
//Http_SendData(c_fd,"application/x-jpg","./html/my.jpg");
//Http_SendPic(c_fd,"application/x-jpg","./html/my.jpg");
Http_Content(c_fd);
}
else if(strstr(buff,"GET /my_32.jpg HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/my_32.jpg");
}
else if(strstr(buff,"GET /100.bmp HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/100.bmp");
}
else if(strstr(buff,"GET /favicon.ico HTTP/1.1"))
{
Http_SendData(c_fd,"image/x-icon","./html/wmp.ico");
}
else
{
Http_SendData(c_fd,"application/x-jpg","./html/limeng.jpg");
//Http_SendData(c_fd,"application/x-bmp","./html/my.jpg");
}
close(c_fd);
//printf("22222222222222222222222 \n");
pthread_exit(NULL);
}
int generate_pic()
{
int fd = open("/dev/video0",O_RDWR); //打开摄像头设备
if (fd < 0)
{
perror("打开设备失败");
return -1;
}
struct v4l2_format vfmt;
vfmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //摄像头采集
vfmt.fmt.pix.width = 640; //设置摄像头采集参数,不可以任意设置
vfmt.fmt.pix.height = 480;
vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG; //设置视频采集格式为mjpg格式
int ret = ioctl(fd,VIDIOC_S_FMT,&vfmt);
if (ret < 0)
{
perror("设置格式失败1");
}
//申请内核空间
struct v4l2_requestbuffers reqbuffer;
reqbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
reqbuffer.count = 4; //申请4个缓冲区
reqbuffer.memory = V4L2_MEMORY_MMAP; //映射方式
ret = ioctl(fd,VIDIOC_REQBUFS,&reqbuffer);
if (ret < 0)
{
perror("申请空间失败");
}
//映射
unsigned char *mptr[4];//保存映射后用户空间的首地址
unsigned int size[4];
struct v4l2_buffer mapbuffer;
//初始化type和index
mapbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
for(int i = 0; i <4;i++) {
mapbuffer.index = i;
ret = ioctl(fd,VIDIOC_QUERYBUF,&mapbuffer); //从内核空间中查询一个空间作映射
if (ret < 0)
{
perror("查询内核空间失败");
}
//映射到用户空间
mptr[i] = (unsigned char *)mmap(NULL,mapbuffer.length,PROT_READ|PROT_WRITE,MAP_SHARED,fd,mapbuffer.m.offset);
size[i] = mapbuffer.length; //保存映射长度用于后期释放
//查询后通知内核已经放回
ret = ioctl(fd,VIDIOC_QBUF,&mapbuffer);
if (ret < 0)
{
perror("放回失败");
}
}
//开始采集
int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd,VIDIOC_STREAMON,&type);
if (ret < 0){ perror("开启失败");}
//定义一个空间存储解码后的rgb
int sum = 0;//这里是用来控制读取的帧的数量 实际场景下 可以不要这个限制
int read_cnt = 0 ;//记录读取图像线程
char save_name[10];
int save_idx = 0 ;
struct timeval start_time, end_time;
while(1)
{
struct v4l2_buffer readbuffer;
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //每个结构体都需要设置type为这个参赛要记住
ret = ioctl(fd,VIDIOC_DQBUF,&readbuffer);
int read_len = mapbuffer.length;
int send_len = 1024;
//printf("%d \n",read_len);
if (ret < 0)
{
perror("读取数据失败");
}
if(1)
{
//void *ptr = mptr[readbuffer.index];
//pthread_mutex_lock(&mutex);
//gettimeofday(&start_time, NULL);
pthread_mutex_lock(&lock);
void *ptr = mptr[readbuffer.index];
///ppp_tr = mptr[readbuffer.index];
//auto beg = clock();
//printf("read frame %d \n",beg);
//printf("read frame %d %d %d \n",read_cnt++,send_html_cnt,beg);
//pthread_mutex_lock(&mutex);
memcpy(jpg_buff, ptr, readbuffer.length);//将读取到的图像拷贝到字符数组
//ppp_tr = ptr ;
//auto end = clock();
//gettimeofday(&end_time, NULL);
//double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
sprintf(save_name,"my_%d.jpg",save_idx++);
//int fd = open(save_name, O_CREAT |O_WRONLY|O_TRUNC,0666);
//char buf[1024*600]={0};
//FILE *file=fopen(save_name, "w");
//fwrite(mptr[readbuffer.index] , readbuffer.length, 1,file);
//fwrite(ptr , readbuffer.length, 1,file);
//close(file);
//write(fd,buf,1024 * 600);
//close(fd);
pthread_mutex_unlock(&lock);
pthread_cond_signal(&hasNode);
//gettimeofday(&end_time, NULL);
//pthread_mutex_unlock(&mutex);
auto end = clock();
//printf("runtime is %f \n", (end - beg) / CLOCKS_PER_SEC );
//printf("runtime is %f \n", (end - beg) );
printf("runtime is %d ms \n", timeuse );
//sleep(20);
usleep(1);
}
//通知内核使用完毕
ret = ioctl(fd, VIDIOC_QBUF, &readbuffer);
if(ret < 0)
{
perror("放回队列失败");
}
}
//停止采集
ret = ioctl(fd,VIDIOC_STREAMOFF,&type);
//释放映射
for(int i=0; i<4; i)munmap(mptr[i], size[i]);
close(fd); //关闭文件
return 0;
err1:
close(fd_fb);
return -1;
}
void shuijiao()
{
struct timeval start_time, end_time;
while(1){
gettimeofday(&start_time, NULL);
pthread_cond_wait(&hasNode,&lock);
usleep(10000);
pthread_mutex_unlock(&lock);
gettimeofday(&end_time, NULL);
double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
//printf("shuijiao time is %f ms \n", timeuse * 1000 );
}
}
int main()
{
int sockfd=socket(AF_INET,SOCK_STREAM,0);
if(sockfd==-1)
{
printf("创建网络套接字失败\n");
return 0;
}
/*允许绑定已使用的端口号*/
int on = 1;
setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
/*绑定端口号*/
struct sockaddr_in s_addr=
{
.sin_family=AF_INET,//IPV4
.sin_port=htons(HTTP_PORT),
.sin_addr.s_addr=INADDR_ANY
};
if(bind(sockfd,(const struct sockaddr *)&s_addr,sizeof(s_addr)))
{
printf("绑定端口号失败\n");
return 0;
}
/*设置监听数量*/
listen(sockfd,100);
/*等待客户端连接*/
struct sockaddr_in c_addr;
socklen_t len=sizeof(c_addr);
int c_fd;
pthread_t pthid;
int *p=NULL;
pthread_t pthid1;
pthread_create(&pthid1,NULL,generate_pic,NULL);
pthread_detach(pthid1);//设置为分离属性
//shuijiao();
//while(1){shuijiao();}
while(1)
{
c_fd=accept(sockfd,(struct sockaddr *)&c_addr,&len);
if(c_fd==-1)
{
printf("客户端连接失败\n");
continue;
}
printf("套接字 : %d 连接成功,%s:%d\n",c_fd,inet_ntoa(c_addr.sin_addr),ntohs(c_addr.sin_port));
p=malloc(4);
*p=c_fd;
pthread_create(&pthid,NULL,pth_work,p);
pthread_detach(pthid);//设置为分离属性
}
}
El comando para compilar el código es
gcc serv.c -o serv -ljpeg