Background
In order to realize the real-time streaming and image processing of UAV video, the video encoding and decoding must be completed first. DJI's video interface is really too pitted! I have consulted many articles of great gods, most of which are decoding local files or directly pulling streams from the server, which cannot achieve the real-time dynamic stream decoding I want. After half a month, I can finally decode in real time. I hope my research results can help more. Multiple people.
- The mainstream video compression format is h264 (IDR encoding), there are many related tutorials, and there are almost no content related to GDR encoding, which cannot be decoded by extracting nalu from a byte stream;
- Directly sample the aircraft video and save it as a local file and cannot be played directly. After being converted to mp4 format, there will be a blurry screen when playing;
- DJI's soft solution interface cpu load is too high.
The H20 main camera video encoding format of DJI M300 is h264 (GDR), and OSDK-4.0 provides three aircraft camera video interface samples.
- OSDK calls ffmpeg to soft-decode the h264 raw stream, sends out the Mat array readable by OpenCV, and directly displays a frame of image;
- Based on the first interface, loop and sleep are added, and the image frame analog video stream is continuously launched;
- Provide a h264 (GDR) bare stream function directly, the outgoing parameter is a data packet address and data packet length.
problem
There is a problem. After initializing the camera-related parameters, the interface provided by DJI will cyclically call back the liveView function pushed by a data packet. The GStreamer plug-in appsrc that imports external data also needs to cyclically call back the need-data function. The two cyclic callback functions run at the same time. To synchronize the data interaction, I designed a circular queue buffer to store public data, and use a mutex lock with condition variables to ensure that the two threads are synchronized.
achieve
Sending end:
/***************************************
@Copyright(C) All rights reserved.
@Author DENG Ganglin
@Date 2020.08
***************************************/
#include "dji_vehicle.hpp"
#include "dji_linux_helpers.hpp"
#include <gst/gst.h>
#include <gst/gstminiobject.h>
#include <glib.h>
#include <stdlib.h>
#include <pthread.h>
#include <iostream>
#include <sys/stat.h>
#include <sys/types.h>
#include <queue>
#include <pthread.h>
#include <unistd.h>
#include "loopqueue.hpp"
using namespace DJI::OSDK;
using namespace std;
// 创建插件
GstElement *pipeline, *appsrc, *parse, *decoder, *scale, *filter1, *conv, *encoder, *filter2, *rtppay, *udpsink;
GstCaps *scale_caps;
GstCaps *omx_caps;
static GMainLoop *loop;
GstBuffer *buf;
// 定义结构体,保存生产者的数据
typedef struct Msg
{
int len;
uint8_t* data;
}msg;
// 创建循环队列
LoopQueue<msg*> loopqueue(256);
pthread_mutex_t myMutex = PTHREAD_MUTEX_INITIALIZER; // 初始化互斥锁
pthread_cond_t cond = PTHREAD_COND_INITIALIZER; // 初始化条件变量
int count_t = 0; // 打印检查数据流动是否正常
static void cb_need_data(GstElement *appsrc, guint size, gpointer user_data)
{
static GstClockTime timestamp = 0;
GstFlowReturn ret;
GstMapInfo map;
count_t++;
// 给互斥量加锁
pthread_mutex_lock(&myMutex);
// 判断条件变量是否满足 访问公共区条件
while (0 == loopqueue.getSize())
{
pthread_cond_wait(&cond, &myMutex);
}
// 访问公共区, 取数据
// 往appsrc输入数据
buf = gst_buffer_new_allocate(NULL, loopqueue.top()->len, NULL);
gst_buffer_map(buf, &map, GST_MAP_WRITE);
memcpy((guchar *)map.data, (guchar *)(loopqueue.top()->data), loopqueue.top()->len);
printf("count_t = %d, bufLen = %d\n", count_t, loopqueue.top()->len);
// 给互斥量解锁
pthread_mutex_unlock(&myMutex);
GST_BUFFER_PTS(buf) = timestamp;
GST_BUFFER_DURATION(buf) = gst_util_uint64_scale_int(1, GST_SECOND, 1);
timestamp += GST_BUFFER_DURATION(buf);
g_signal_emit_by_name(appsrc, "push-buffer", buf, &ret);
gst_buffer_unref(buf);
if (ret != 0)
{
g_main_loop_quit(loop);
}
// 释放摘下的结点
delete loopqueue.top()->data;
delete loopqueue.top();
loopqueue.pop();
}
void liveViewSampleCb(uint8_t* buf264, int bufLen, void* userData)
{
// 生产数据
msg *mp = new msg;
mp->len = bufLen;
mp->data = new uint8_t[bufLen];
memcpy(mp->data, buf264, bufLen);
// 加锁
pthread_mutex_lock(&myMutex);
// 访问公共区
loopqueue.push(mp);
// 解锁
pthread_mutex_unlock(&myMutex);
// 唤醒阻塞在条件变量上的消费者
pthread_cond_signal(&cond);
return;
}
// 创建管道，触发need-data信号
void *consumer(void *arg)
{
// init GStreamer
gst_init(NULL, NULL);
loop = g_main_loop_new(NULL, FALSE);
// setup pipeline
pipeline = gst_pipeline_new("pipeline");
appsrc = gst_element_factory_make("appsrc", "source");
parse = gst_element_factory_make("h264parse", "parse");
decoder = gst_element_factory_make("avdec_h264", "decoder");
scale = gst_element_factory_make("videoscale", "scale");
filter1 = gst_element_factory_make("capsfilter", "filter1");
conv = gst_element_factory_make("videoconvert", "conv");
encoder = gst_element_factory_make("omxh264enc", "encoder");
filter2 = gst_element_factory_make("capsfilter", "filter2");
rtppay = gst_element_factory_make("rtph264pay", "rtppay");
udpsink = gst_element_factory_make("udpsink", "sink");
scale_caps = gst_caps_new_simple("video/x-raw",
"width", G_TYPE_INT, 1280,
"height", G_TYPE_INT, 720,
NULL);
omx_caps = gst_caps_new_simple("video/x-h264",
"stream-format", G_TYPE_STRING, "byte-stream",
NULL);
g_object_set(G_OBJECT(filter1), "caps", scale_caps, NULL);
g_object_set(G_OBJECT(filter2), "caps", omx_caps, NULL);
gst_caps_unref(scale_caps);
gst_caps_unref(omx_caps);
g_object_set(udpsink, "host", "127.0.0.1", NULL);
g_object_set(udpsink, "port", 5555, NULL);
g_object_set(udpsink, "sync", false, NULL);
g_object_set(udpsink, "async", false, NULL);
// setup
//g_object_set(G_OBJECT(appsrc), "caps",
// gst_caps_new_simple("video/x-raw",
// "stream-format", G_TYPE_STRING, "byte-stream",
// NULL), NULL);
gst_bin_add_many(GST_BIN(pipeline), appsrc, parse, decoder, scale, filter1, conv, encoder, filter2, rtppay, udpsink, NULL);
gst_element_link_many(appsrc, parse, decoder, scale, filter1, conv, encoder, filter2, rtppay, udpsink, NULL);
// setup appsrc
//g_object_set(G_OBJECT(appsrc),
// "stream-type", 0,
// "format", GST_FORMAT_TIME, NULL);
g_signal_connect(appsrc, "need-data", G_CALLBACK(cb_need_data), NULL);
/* play */
std::cout << "start sender pipeline" << std::endl;
gst_element_set_state(pipeline, GST_STATE_PLAYING);
g_main_loop_run(loop);
/* clean up */
gst_element_set_state(pipeline, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(pipeline));
g_main_loop_unref(loop);
gst_buffer_unref(buf);
}
// 初始化飞机开始推送264码流数据包
void *producer(void *arg)
{
while (true)
{
// 启动 OSDK.
bool enableAdvancedSensing = true;
LinuxSetup linuxEnvironment(1, NULL, enableAdvancedSensing);
Vehicle *vehicle = linuxEnvironment.getVehicle();
if (vehicle == nullptr)
{
std::cout << "Vehicle not initialized, exiting.\n";
return NULL;
}
vehicle->advancedSensing->startH264Stream(LiveView::OSDK_CAMERA_POSITION_NO_1,
liveViewSampleCb,
NULL);
sleep(100); //可根据需求定义回调函数一次调用的时间
}
}
int main(int argc, char** argv)
{
// 创建一个生产者线程和一个消费者线程
int rc1, rc2;
pthread_t pid, cid;
if (rc1 = pthread_create(&pid, NULL, producer, NULL))
cout << "producer thread creation failed: " << rc1 << endl;
if (rc2 = pthread_create(&cid, NULL, consumer, NULL))
cout << "consumer thread creation failed: " << rc2 << endl;
pthread_join(pid, NULL);
pthread_join(cid, NULL);
pthread_mutex_destroy(&myMutex);
pthread_cond_destroy(&cond);
return 0;
}
Receiving end:
/***************************************
@Copyright(C) All rights reserved.
@Author DENG Ganglin
@Date 2020.08
***************************************/
#include "opencv2/opencv.hpp"
#include <cstdio>
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
VideoCapture cap("udpsrc port=5555 ! application/x-rtp,encoding-name=H264 ! rtpjitterbuffer latency=0 ! rtph264depay ! avdec_h264 ! videoconvert ! appsink");
if( !cap.isOpened())
{
cout<<"Fail"<<endl;
return -1;
}
namedWindow("frame",1);
for(;;)
{
Mat frame(1280,720, CV_8UC3, Scalar(0,0,255));
cap >> frame; // get a new frame from camera
imshow("frame", frame);
if(waitKey(30) >= 0) break;
}
return 0;
}
to sum up
- There is no hpp code for the circular queue, it is not original, there are many on the Internet;
- A stupid little mistake, applying for memory new uint8_t[bufLen] is written as new uint8_t(bufLen), which is equivalent to only applying for a uint8_t memory, and the program keeps reporting illegal memory when it runs. It should not be;
- The measured sender can only use soft coding for the first encoding after parsing the bare stream, and hard decoding can be used for subsequent decoding. The cpu load is about 180%, which is slightly lower than the full soft solution 250%;
- The actual effect is not bad, the delay is about 800ms, and it can be optimized later.