Recording audio and video output class encapsulation

illustrate

The modular design idea is very important, which greatly reduces repetitive meaningless labor and does not waste time repeatedly on the knowledge already mastered. This article records the audio and video output class packaged by myself, instantiates the class directly, first sets the corresponding audio and video parameters, and then calls the interface to pass in the decoded audio and video data to play. The packaging class is mainly designed with Qt + FFmpeg + SDL.

Video output class

The video output is the data rendering after image decoding. The method is not unique. For example, an existing frame of videoFrame decoded by FFmpeg is in the format of yuv420P.

First of all, it can be realized with the functions of Qt itself:
1. CPU method, convert the format of videoFrame to RGB32, construct QImage and send it to the interface thread for rendering
2. GPU method, use QOpenglWidget to directly render videoFrame of yuv420P.
The obvious advantage of using Qt method is that it is not Three-party libraries need to be introduced to achieve two rendering methods. The disadvantage is that the amount of code is relatively large, especially in the CPU mode, which needs to be format converted and sent to the interface thread, and there are differences in code logic, which makes it inconvenient to switch flexibly between the two modes.

Therefore, I personally prefer to use the mature and proven cross-platform SDL library. Although the SDL library is introduced in addition, the disadvantages can be solved perfectly. It also supports CPU and GPU rendering, and only needs one parameter to switch modes, which is very flexible. And observe that the rendering performance of SDL should be slightly better than the Qt way. SDL rendering AVFrame is described in detail in my previous blog post

audio output class

The audio output is to play the decoded pcm data. The method is not unique. Compared with video, the data volume of audio is much smaller, and the processing method is simpler. You can use SDL to do it, but the audio of SDL is generally through the internal callback method, and the logic will be a little complicated. The audio output interface of Qt belongs to the synchronous logic, which is more suitable.

Encapsulation class code

The following is the encapsulated class code, complete the basic environment configuration, copy and use it directly. Example of video output:
1.setShowHandle set playback handle
2.setVideoParm set playback parameters
3.startDisplay continue to fill video decoding data

1. Header file mediaoutput.h

#ifndef MEDIAOUTPUT_H
#define MEDIAOUTPUT_H

#include <QAudioFormat>
#include <QAudioOutput>
#include <QDebug>
#include "ffmpegheader.h"
#include "SDL.h"

//视频输出类 SDL
class VideoOutput
{
    
    
public:
    VideoOutput();
    ~VideoOutput();

    //只显示不负责释放 AVFrame为YUV420P格式
    void startDisplay(AVFrame *videoFrame);
    void stopDisplay();

    //设置渲染窗口句柄
    void setShowHandle(void *id);
    //设置视频宽、高、渲染方式
    bool setVideoParm(int width, int height, bool cpuMode = true);
    //释放
    void release();

private:
    //初始化状态
    bool hasInit;

    //用到的相关变量
    SDL_Window *sdlWindow;
    SDL_Renderer *sdlRender;
    SDL_Texture *sdlTexture;

    //显示的句柄
    void *showId;
};


//音频输出类 Qt自带
class AudioOutput
{
    
    
public:
    AudioOutput();
    ~AudioOutput();

    //只播放不负责释放 AVFrame为解码后音频帧
    void startPlay(AVFrame *audioFrame);

    //设置音频参数 采样率 采样格式 通道数 均为ffmpeg解析到的参数
    bool setAudioParm(int sampleRate, int sampleFmt, int channels);
    //释放
    void release();

private:
    //初始化状态
    bool hasInit;

    //音频参数
    int sampleRate;
    int sampleSize;
    int channels;

    //设备输出
    QAudioOutput *audioOutput;
    QIODevice *audioDevice;

    //重采样
    SwrContext *audioSwrCtx;
    uint8_t *audioData;
};

#endif // MEDIAOUTPUT_H

2. Implement the file mediaoutput.cpp

#include "mediaoutput.h"

VideoOutput::VideoOutput()
{
    
    
    sdlWindow = nullptr;
    sdlRender = nullptr;
    sdlTexture = nullptr;
    showId = nullptr;

    hasInit = false;
}

VideoOutput::~VideoOutput()
{
    
    
    release();
}

void VideoOutput::startDisplay(AVFrame *videoFrame)
{
    
    
    if(!hasInit)
    {
    
    
        qDebug() << "视频参数未设置";
        return;
    }

    if(videoFrame == nullptr)
    {
    
    
        qDebug() << "视频帧无效";
        return;
    }

    //句柄尺寸变化 那么SDL渲染的窗口也会跟着变化
    SDL_Surface *sdlSurface = SDL_GetWindowSurface(sdlWindow);
    if (!sdlSurface)
    {
    
    
        qDebug() << "SDL_GetWindowSurface fail" << SDL_GetError();
        return;
    }

    //渲染
    int ret = SDL_UpdateYUVTexture(sdlTexture, NULL, videoFrame->data[0], videoFrame->linesize[0], videoFrame->data[1], videoFrame->linesize[1], videoFrame->data[2], videoFrame->linesize[2]);
    if (ret != 0)
        qDebug() << "SDL_UpdateYUVTexture fail" << SDL_GetError();

    ret = SDL_RenderClear(sdlRender);
    if (ret != 0)
        qDebug() << "SDL_RenderClear fail" << SDL_GetError();

    ret = SDL_RenderCopy(sdlRender, sdlTexture, NULL, NULL);
    if (ret != 0)
        qDebug() << "SDL_RenderCopy fail" << SDL_GetError();

    SDL_RenderPresent(sdlRender);
}

void VideoOutput::stopDisplay()
{
    
    
    if(!hasInit)
        return;

    SDL_SetRenderDrawColor(sdlRender, 240, 240, 240, 100);
    SDL_RenderClear(sdlRender);
    SDL_RenderPresent(sdlRender);
}

void VideoOutput::setShowHandle(void *id)
{
    
    
    this->showId = id;
}

bool VideoOutput::setVideoParm(int width, int height, bool cpuMode)
{
    
    
    if(this->showId == nullptr)
    {
    
    
        qDebug() << "video showId is nullptr";
        return false;
    }

    //SDL存在界面伸缩有时画面卡住 查阅资料应该是与SDL内部逻辑处理冲突 多次测试采用重置参数方法更为直接有效
    release();

    //默认cpu软件渲染模式 相比gpu硬件渲染兼容性更高
    SDL_RendererFlags flag;
    if(cpuMode)
        flag = SDL_RENDERER_SOFTWARE;
    else
        flag = SDL_RENDERER_ACCELERATED;

    sdlWindow = SDL_CreateWindowFrom(showId);
    if(!sdlWindow)
    {
    
    
        qDebug() << "SDL_CreateWindowFrom error" << SDL_GetError();
        return false;
    }

    sdlRender = SDL_CreateRenderer(sdlWindow, -1, flag);
    if (!sdlRender)
    {
    
    
        qDebug() << "SDL_CreateRenderer error" << SDL_GetError();
        return false;
    }

    sdlTexture = SDL_CreateTexture(sdlRender, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING, width, height);
    if (!sdlTexture)
    {
    
    
        qDebug() << "SDL_CreateRenderer error" << SDL_GetError();
        return false;
    }

    SDL_ShowWindow(sdlWindow);
    hasInit = true;
    return true;
}

void VideoOutput::release()
{
    
    
    if(sdlTexture)
    {
    
    
        SDL_DestroyTexture(sdlTexture);
        sdlTexture = nullptr;
    }

    if(sdlRender)
    {
    
    
        SDL_DestroyRenderer(sdlRender);
        sdlRender = nullptr;
    }

    if(sdlWindow)
    {
    
    
        SDL_DestroyWindow(sdlWindow);
        sdlWindow = nullptr;
    }

    hasInit = false;
}


AudioOutput::AudioOutput()
{
    
    
    audioOutput = nullptr;
    audioDevice = nullptr;
    audioSwrCtx = nullptr;
    audioData = nullptr;
    sampleRate = 0;
    sampleSize = 0;
    channels = 0;
    hasInit = false;
}

AudioOutput::~AudioOutput()
{
    
    
    release();
}

void AudioOutput::startPlay(AVFrame *audioFrame)
{
    
    
    if(!hasInit)
    {
    
    
        qDebug() << "音频参数未设置";
        return;
    }

    if(audioFrame == nullptr)
    {
    
    
        qDebug() << "音频帧无效";
        return;
    }

    int outSize = av_samples_get_buffer_size(NULL, channels, audioFrame->nb_samples, AV_SAMPLE_FMT_S16, 1);

    int result = swr_convert(audioSwrCtx, &audioData, outSize, (const uint8_t **)audioFrame->data, audioFrame->nb_samples);
    if(result >= 0)
        audioDevice->write((char *)audioData, outSize);

}

bool AudioOutput::setAudioParm(int sampleRate, int sampleFmt, int channels)
{
    
    
    //释放上次重置即可
    release();

    this->sampleRate = sampleRate;
    this->sampleSize = av_get_bytes_per_sample((AVSampleFormat)sampleFmt) / channels;
    this->channels = channels;

    QAudioFormat format;
    format.setCodec("audio/pcm");
    format.setSampleRate(sampleRate);
    format.setSampleSize(sampleSize * 8);
    format.setChannelCount(channels);
    format.setSampleType(QAudioFormat::SignedInt);
    format.setByteOrder(QAudioFormat::LittleEndian);

    QAudioDeviceInfo info(QAudioDeviceInfo::defaultOutputDevice());

    bool res = info.isFormatSupported(format);
    if(!res)
    {
    
    
        qDebug() << "AudioFormat not Support";
        return false;
    }

    audioOutput = new QAudioOutput(QAudioDeviceInfo::defaultOutputDevice(), format);
    //设置缓存避免播放音频卡顿  太大可能会分配内存失败导致崩溃
    audioOutput->setBufferSize(40960);
    audioOutput->setVolume(1.0);
    audioDevice = audioOutput->start();

    //设置音频转换
    audioSwrCtx = swr_alloc();
    int64_t channelIn = av_get_default_channel_layout(channels);
    int64_t channelOut = channelIn;
    audioSwrCtx = swr_alloc_set_opts(audioSwrCtx, channelOut, AV_SAMPLE_FMT_S16, sampleRate, channelIn, (AVSampleFormat)sampleFmt, sampleRate, 0, 0);
    if(0 > swr_init(audioSwrCtx))
    {
    
    
        qDebug() << "audio swr_init error";
        release();
        return false;
    }

    //分配音频数据内存19200 可调节
    audioData = (uint8_t *)av_malloc(19200 * sizeof(uint8_t));

    hasInit = true;
    return true;
}

void AudioOutput::release()
{
    
    
    if(audioDevice)
        audioDevice->close();
    if(audioSwrCtx)
        swr_free(&audioSwrCtx);
    if(audioData)
        av_free(audioData);
    if(audioOutput)
        audioOutput->deleteLater();

    hasInit = false;
}