Project scenario:
项目需求，需要做一个基于边缘端的人脸识别远程监控摄像头并在网页前端展示 ，这里采用国产香橙派作为边缘计算终端，安装ubuntu系统，系统中采用v4l2接口对摄像头进行获取，当客户端通过网页进行请求时，服务器通过http服务的形式将一帧帧图像发送给客户端，只要一秒钟能够传送25帧左右，展示效果就是在网页端播放视频：
Problem description 1
How to get frame data from the camera , here we connect the USB camera to the development board:
It can be seen that there is indeed a video0 camera, which is an external USB camera
Solution 1:
Use the V4L2 interface to map the data read from the kernel state to the user space through interrupts:
The following code maps kernel space to user space
for(int i = 0; i <4;i++) {
mapbuffer.index = i;
ret = ioctl(fd,VIDIOC_QUERYBUF,&mapbuffer); //从内核空间中查询一个空间作映射
if (ret < 0)
{
perror("查询内核空间失败");
}
//映射到用户空间
mptr[i] = (unsigned char *)mmap(NULL,mapbuffer.length,PROT_READ|PROT_WRITE,MAP_SHARED,fd,mapbuffer.m.offset);
size[i] = mapbuffer.length; //保存映射长度用于后期释放
//查询后通知内核已经放回
ret = ioctl(fd,VIDIOC_QBUF,&mapbuffer);
if (ret < 0)
{
perror("放回失败");
}
}
The following code obtains data in the kernel mode through interrupts, and copies the data in memory in the user mode, that is, copies the data to an array, so that other threads can perform data processing and data display
struct v4l2_buffer readbuffer;
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //每个结构体都需要设置type为这个参赛要记住
ret = ioctl(fd,VIDIOC_DQBUF,&readbuffer);
int read_len = mapbuffer.length;
int send_len = 1024;
//printf("%d \n",read_len);
if (ret < 0)
{
perror("读取数据失败");
}
if(1)
{
void *ptr = mptr[readbuffer.index];
memcpy(jpg_buff, ptr, readbuffer.length);
}Problem description 2
To play the video on the front end of the webpage, the video is first composed of frames of images, here we send a frame every short interval, and the http client and server adopt the keep-live long connection form. In order to speed up, here I use two threads, ready-made 1 reads the video, and puts the image frame in a public array, and thread 2 sends the data in the public array to the client through tcp transmission.
At the same time, a condition variable is used for synchronization between thread 1 and thread
Solution 2:
Multiple threads share global variables to speed up video reading and image sending:
thread 1
pthread_mutex_lock(&lock);
void *ptr = mptr[readbuffer.index];
memcpy(jpg_buff, ptr, readbuffer.length);//将读取到的图像拷贝到字符数组
pthread_mutex_unlock(&lock);
pthread_cond_signal(&hasNode);
thread 2
pthread_cond_wait(&hasNode,&lock);
void *ppptr = jpg_buff;
pthread_mutex_unlock(&lock);The jpg_buff here is a shared global array
Problem description 3
Web page long connection setting problem :
When the server returns the message header, add
"Content-Type:multipart/x-mixed-replace;
That means the current connection is based on a long connection
solution:
Tip: Fill in the specific solution to the problem here:
The following is the content of the header
/*
HTTP长连接处理客户端请求
"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n"
*/This is the effect picture when the program is running:
You need to type in the URL on the terminal on the same LAN as the development board: 192.168.0.105:8080
Where 192.168.0.105 is the ip address of the development board
Finally attach the total code
#include <stdio.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include <assert.h>
#include <fcntl.h>
#include <jpeglib.h>
#include <linux/fb.h>
#include <linux/videodev2.h>
#include <netinet/in.h>
#include <netinet/ip.h> /* superset of previous */
#include <pthread.h>
#include "sd_usb_pic.h"
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/types.h> /* See NOTES */
#include <unistd.h>
#include <sys/time.h>
#define HTTP_PORT 8080 //HTTP服务器端口号
pthread_mutex_t mutex;
pthread_cond_t hasNode = PTHREAD_COND_INITIALIZER;
pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
char jpg_buff[614400]; //保存从USB摄像头读取的图像矩阵
int send_html_cnt = 0 ;
/*
服务端响应客户端请求
"HTTP/1.1 200 OK\r\n"
"Content-type:image/jpeg\r\n"
"Content-Length:1234\r\n"
"\r\n"
形参：c_fd --客户端套接字
type --文件类型
file --要发送的文件
返回值：0成功，其它失败
*/
void *ppp_tr = NULL;
int fd_fb;
static struct fb_var_screeninfo var; /* LCD可变参数 */
static unsigned int *fb_base = NULL; /* Framebuffer映射基地址 */
int lcd_w = 800 ,lcd_h= 480; //定义显示器分辨率
/*
形参：c_fd --客户端套接字
type --文件类型
file --要发送的文件
返回值：0成功，其它失败
*/
int Http_SendData(int c_fd,const char *type,const char *file)
{
int fd=open(file,O_RDONLY);//打开文件
if(fd<0)return -1;//打开文件失败
struct stat statbuf;
fstat(fd,&statbuf);
if(statbuf.st_size<=0)
{
close(fd);
return -2;
}
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Content-type:%s\r\n"
"Content-Length:%ld\r\n"
"\r\n",type,statbuf.st_size);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
close(fd);
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
while(1)
{
size=read(fd,buff,sizeof(buff));
if(write(c_fd,buff,size)!=size)break;//发送失败
if(size!=sizeof(buff))break;//发送完成
}
close(fd);
return 0;
}
int Http_SendPic(int c_fd,const char *type,const char *file)
{
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Content-type:%s\r\n"
"Content-Length:%ld\r\n"
"\r\n",type,614400);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
int cnt = 0 ;
pthread_mutex_lock(&mutex);
void *ppptr = jpg_buff;
while(1)
{
int size= 1024;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
pthread_mutex_unlock(&mutex);
return 0;
}
int Http_SendPic1(int c_fd,const char *type,const char *file)
{
char buff[1024]={0};
snprintf(buff,sizeof(buff),"HTTP/1.1 200 OK\r\n"
"Server:LiMeng \r\n"
"Content-type:image/jpeg\r\n"
"Content-Length:%ld\r\n"
"\r\n",614400);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -3;//发送数据头失败
}
/*发送文件内容*/
int size;
int cnt = 0 ;
pthread_mutex_lock(&mutex);
void *ppptr = jpg_buff;
while(1)
{
int size= 1024;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
pthread_mutex_unlock(&mutex);
return 0;
}
/*
HTTP长连接处理客户端请求
"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n"
*/
int Http_Content(int c_fd)
{
char buff[1024]={0};
/*建立长连接*/
snprintf(buff,sizeof(buff),"HTTP/1.0 200 OK\r\n"
"Server: wbyq\r\n"
"Content-Type:multipart/x-mixed-replace;boundary=boundarydonotcross\r\n"
"\r\n"
"--boundarydonotcross\r\n");
if(write(c_fd,buff,strlen(buff))!=strlen(buff))return -1;//发送报文头失败
int jpe_image_size = 614400;//保存jpeg图像大小
//int send_html_cnt = 0 ;
clock_t t1;
t1 = clock();
char save_name[10];
int save_idx = 0 ;
struct timeval start_time, end_time;
while(1)
{
//auto beg = clock();
t1 = clock();
pthread_cond_wait(&hasNode,&lock);
//printf("wait time is %d us \n", (clock() - t1) );
auto beg = clock();
gettimeofday(&start_time, NULL);
/*
(1)响应报文头
"Content-type:image/jpeg\r\n"
"Content-Length:666\r\n"
"\r\n"
*/
snprintf(buff,sizeof(buff), "Content-type:image/jpeg\r\n"
"Content-Length:%d\r\n"
"\r\n",jpe_image_size);
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
return -2;//响应报文头失败
}
/*发送jpg图像数据*/
//pthread_mutex_lock(&mutex);//互斥锁上锁
//pthread_cond_wait(&hasNode,&lock);
void *ppptr = jpg_buff;
//void *ppptr = ppp_tr;
//sprintf(save_name,"my_%d.jpg",save_idx++);
//FILE *file=fopen(save_name, "w");
//fwrite(ppptr , 614400, 1,file);
//close(file);
int cnt = 0 ;
while(1)
{
int size= 1024 * 600;
//int size= 1024 ;
int wt_len = write(c_fd,ppptr,size);//发送失败
cnt += wt_len ;
ppptr += wt_len ;
if(cnt >= 614400 ) {break;}//发送完成
}
//pthread_mutex_unlock(&mutex);//互斥锁上锁
//pthread_mutex_unlock(&lock);
//sleep(20);
/*
(3)发送间隔字符串
"\r\n"
"--boundarydonotcross\r\n"
*/
strcpy(buff,"\r\n--boundarydonotcross\r\n");
if(write(c_fd,buff,strlen(buff))!=strlen(buff))
{
break;//发送间隔符失败
}
auto end = clock();
gettimeofday(&end_time, NULL);
double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
//printf("sendtime is %d us \n", (end - beg) );
//printf("send cnt is %d us \n", send_html_cnt );
send_html_cnt++;
//pthread_mutex_unlock(&lock);
//usleep(40000);
//auto beg = clock();
//printf("sendtime is %d ms \n", (end - beg) );
printf("sendtime is %d ms \n", timeuse );
pthread_mutex_unlock(&lock);
usleep(1);
}
return -4;//发送图像数据失败
}
/*线程工作函数*/
void *pth_work(void *arg)
{
int c_fd=*(int *)arg;
free(arg);
char buff[1024]={0};
int size;
size=read(c_fd,buff,sizeof(buff)-1);
if(size<=0)
{
close(c_fd);
pthread_exit(NULL);
}
buff[size]='\0';
printf("buff=%s\n",buff);
if(strstr(buff,"GET / HTTP/1.1"))//请求网页文件
{
Http_SendData(c_fd,"text/html","./html/image.html");
}
else if(strstr(buff,"GET /1.bmp HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/1.bmp");
}
else if(strstr(buff,"GET /my.jpg HTTP/1.1"))
{
//Http_SendData(c_fd,"application/x-jpg","./html/my.jpg");
//Http_SendPic(c_fd,"application/x-jpg","./html/my.jpg");
Http_Content(c_fd);
}
else if(strstr(buff,"GET /my_32.jpg HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/my_32.jpg");
}
else if(strstr(buff,"GET /100.bmp HTTP/1.1"))
{
Http_SendData(c_fd,"application/x-bmp","./html/100.bmp");
}
else if(strstr(buff,"GET /favicon.ico HTTP/1.1"))
{
Http_SendData(c_fd,"image/x-icon","./html/wmp.ico");
}
else
{
Http_SendData(c_fd,"application/x-jpg","./html/limeng.jpg");
//Http_SendData(c_fd,"application/x-bmp","./html/my.jpg");
}
close(c_fd);
//printf("22222222222222222222222 \n");
pthread_exit(NULL);
}
int generate_pic()
{
int fd = open("/dev/video0",O_RDWR); //打开摄像头设备
if (fd < 0)
{
perror("打开设备失败");
return -1;
}
struct v4l2_format vfmt;
vfmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //摄像头采集
vfmt.fmt.pix.width = 640; //设置摄像头采集参数，不可以任意设置
vfmt.fmt.pix.height = 480;
vfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG; //设置视频采集格式为mjpg格式
int ret = ioctl(fd,VIDIOC_S_FMT,&vfmt);
if (ret < 0)
{
perror("设置格式失败1");
}
//申请内核空间
struct v4l2_requestbuffers reqbuffer;
reqbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
reqbuffer.count = 4; //申请4个缓冲区
reqbuffer.memory = V4L2_MEMORY_MMAP; //映射方式
ret = ioctl(fd,VIDIOC_REQBUFS,&reqbuffer);
if (ret < 0)
{
perror("申请空间失败");
}
//映射
unsigned char *mptr[4];//保存映射后用户空间的首地址
unsigned int size[4];
struct v4l2_buffer mapbuffer;
//初始化type和index
mapbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
for(int i = 0; i <4;i++) {
mapbuffer.index = i;
ret = ioctl(fd,VIDIOC_QUERYBUF,&mapbuffer); //从内核空间中查询一个空间作映射
if (ret < 0)
{
perror("查询内核空间失败");
}
//映射到用户空间
mptr[i] = (unsigned char *)mmap(NULL,mapbuffer.length,PROT_READ|PROT_WRITE,MAP_SHARED,fd,mapbuffer.m.offset);
size[i] = mapbuffer.length; //保存映射长度用于后期释放
//查询后通知内核已经放回
ret = ioctl(fd,VIDIOC_QBUF,&mapbuffer);
if (ret < 0)
{
perror("放回失败");
}
}
//开始采集
int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
ret = ioctl(fd,VIDIOC_STREAMON,&type);
if (ret < 0){ perror("开启失败");}
//定义一个空间存储解码后的rgb
int sum = 0;//这里是用来控制读取的帧的数量 实际场景下 可以不要这个限制
int read_cnt = 0 ;//记录读取图像线程
char save_name[10];
int save_idx = 0 ;
struct timeval start_time, end_time;
while(1)
{
struct v4l2_buffer readbuffer;
readbuffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; //每个结构体都需要设置type为这个参赛要记住
ret = ioctl(fd,VIDIOC_DQBUF,&readbuffer);
int read_len = mapbuffer.length;
int send_len = 1024;
//printf("%d \n",read_len);
if (ret < 0)
{
perror("读取数据失败");
}
if(1)
{
//void *ptr = mptr[readbuffer.index];
//pthread_mutex_lock(&mutex);
//gettimeofday(&start_time, NULL);
pthread_mutex_lock(&lock);
void *ptr = mptr[readbuffer.index];
///ppp_tr = mptr[readbuffer.index];
//auto beg = clock();
//printf("read frame %d \n",beg);
//printf("read frame %d %d %d \n",read_cnt++,send_html_cnt,beg);
//pthread_mutex_lock(&mutex);
memcpy(jpg_buff, ptr, readbuffer.length);//将读取到的图像拷贝到字符数组
//ppp_tr = ptr ;
//auto end = clock();
//gettimeofday(&end_time, NULL);
//double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
sprintf(save_name,"my_%d.jpg",save_idx++);
//int fd = open(save_name, O_CREAT |O_WRONLY|O_TRUNC,0666);
//char buf[1024*600]={0};
//FILE *file=fopen(save_name, "w");
//fwrite(mptr[readbuffer.index] , readbuffer.length, 1,file);
//fwrite(ptr , readbuffer.length, 1,file);
//close(file);
//write(fd,buf,1024 * 600);
//close(fd);
pthread_mutex_unlock(&lock);
pthread_cond_signal(&hasNode);
//gettimeofday(&end_time, NULL);
//pthread_mutex_unlock(&mutex);
auto end = clock();
//printf("runtime is %f \n", (end - beg) / CLOCKS_PER_SEC );
//printf("runtime is %f \n", (end - beg) );
printf("runtime is %d ms \n", timeuse );
//sleep(20);
usleep(1);
}
//通知内核使用完毕
ret = ioctl(fd, VIDIOC_QBUF, &readbuffer);
if(ret < 0)
{
perror("放回队列失败");
}
}
//停止采集
ret = ioctl(fd,VIDIOC_STREAMOFF,&type);
//释放映射
for(int i=0; i<4; i)munmap(mptr[i], size[i]);
close(fd); //关闭文件
return 0;
err1:
close(fd_fb);
return -1;
}
void shuijiao()
{
struct timeval start_time, end_time;
while(1){
gettimeofday(&start_time, NULL);
pthread_cond_wait(&hasNode,&lock);
usleep(10000);
pthread_mutex_unlock(&lock);
gettimeofday(&end_time, NULL);
double timeuse = (end_time.tv_sec - start_time.tv_sec) + (double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
//printf("shuijiao time is %f ms \n", timeuse * 1000 );
}
}
int main()
{
int sockfd=socket(AF_INET,SOCK_STREAM,0);
if(sockfd==-1)
{
printf("创建网络套接字失败\n");
return 0;
}
/*允许绑定已使用的端口号*/
int on = 1;
setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
/*绑定端口号*/
struct sockaddr_in s_addr=
{
.sin_family=AF_INET,//IPV4
.sin_port=htons(HTTP_PORT),
.sin_addr.s_addr=INADDR_ANY
};
if(bind(sockfd,(const struct sockaddr *)&s_addr,sizeof(s_addr)))
{
printf("绑定端口号失败\n");
return 0;
}
/*设置监听数量*/
listen(sockfd,100);
/*等待客户端连接*/
struct sockaddr_in c_addr;
socklen_t len=sizeof(c_addr);
int c_fd;
pthread_t pthid;
int *p=NULL;
pthread_t pthid1;
pthread_create(&pthid1,NULL,generate_pic,NULL);
pthread_detach(pthid1);//设置为分离属性
//shuijiao();
//while(1){shuijiao();}
while(1)
{
c_fd=accept(sockfd,(struct sockaddr *)&c_addr,&len);
if(c_fd==-1)
{
printf("客户端连接失败\n");
continue;
}
printf("套接字 : %d 连接成功,%s:%d\n",c_fd,inet_ntoa(c_addr.sin_addr),ntohs(c_addr.sin_port));
p=malloc(4);
*p=c_fd;
pthread_create(&pthid,NULL,pth_work,p);
pthread_detach(pthid);//设置为分离属性
}
}
The command to compile the code is
gcc serv.c -o serv -ljpeg