探索OpenCV的光流算法视频超分与源码实现

在OpenCV4.0以后，视频图像超分模块已经迁移到opencv_contrib独立仓库。在视频超分有两种形式：结合光流算法实现超分、使用CNN卷积神经网络实现超分。在这里主要探索光流算法实现视频超分，然后进一步分析源码实现。

一、视频超分示例

1、光流算法选择

OpenCV提供多种光流算法：farneback、tvl1、brox、pyrlk。同时支持GPU加速，示例代码如下：

static Ptr<cv::superres::DenseOpticalFlowExt> createOptFlow(const string& name, bool useGpu)
{
    if (name == "farneback")
    {
        if (useGpu) // 开启GPU加速，使用CUDA实现
            return cv::superres::createOptFlow_Farneback_CUDA();
        else
            return cv::superres::createOptFlow_Farneback();
    }
    else if (name == "tvl1")
    {
        if (useGpu)
            return cv::superres::createOptFlow_DualTVL1_CUDA();
        else
            return cv::superres::createOptFlow_DualTVL1();
    }
    else if (name == "brox")
        return cv::superres::createOptFlow_Brox_CUDA();
    else if (name == "pyrlk")
        return cv::superres::createOptFlow_PyrLK_CUDA();
    else
        cerr << "Incorrect Optical Flow algorithm - " << name << endl;

    return Ptr<cv::superres::DenseOpticalFlowExt>();
}

2、创建超分实例

根据是否使用CUDA进行GPU加速，来创建视频超分实例：

Ptr<SuperResolution> createSuperResolution(bool useCuda)
{
    if (useCuda)
        return createSuperResolution_BTVL1_CUDA();
    else
        return createSuperResolution_BTVL1();
}

3、执行视频超分

在创建光流算法、视频超分实例后，设置相关参数，执行视频超分操作。示例代码如下：

#include <iostream>
#include <string>
#include <ctype.h>

#include "opencv2/core.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/superres.hpp"
#include "opencv2/superres/optical_flow.hpp"
#include "opencv2/opencv_modules.hpp"

using namespace std;
using namespace cv;
using namespace cv::superres;


int main(int argc, const char* argv[])
{
    CommandLineParser cmd(argc, argv,
        "{ v video      |           | Input video (mandatory)}"
        "{ o output     |           | Output video }"
        "{ s scale      | 4         | Scale factor }"
        "{ i iterations | 180       | Iteration count }"
        "{ t temporal   | 4         | Radius of the temporal search area }"
        "{ f flow       | farneback | Optical flow algorithm (farneback, tvl1, brox, pyrlk) }"
        "{ g gpu        | false     | CPU as default device, cuda for CUDA }"
    );
    // 解析参数
    const string inputVideoName  = cmd.get<string>("video");
    const string outputVideoName = cmd.get<string>("output");
    const int scale              = cmd.get<int>("scale");
    const int iterations         = cmd.get<int>("iterations");
    const int temporalAreaRadius = cmd.get<int>("temporal");
    const string optFlow         = cmd.get<string>("flow");
    string gpuOption             = cmd.get<string>("gpu");
    bool useCuda                 = gpuOption.compare("cuda") == 0;
    // 创建视频超分实例
    Ptr<SuperResolution> superRes = createSuperResolution(useCuda);
    // 创建光流算法
    Ptr<cv::superres::DenseOpticalFlowExt> of = createOptFlow(optFlow, useCuda);
    // 设置光流算法、超分倍数、迭代次数、半径系数
    superRes->setOpticalFlow(of);
    superRes->setScale(scale);
    superRes->setIterations(iterations);
    superRes->setTemporalAreaRadius(temporalAreaRadius);
    // 读取视频帧
    Ptr<FrameSource> frameSource;
    if (useCuda)
    {
        try
        {
            frameSource = createFrameSource_Video_CUDA(inputVideoName);
            Mat frame;
            frameSource->nextFrame(frame);
        }
        catch (const cv::Exception&)
        {
            frameSource.release();
        }
    }
    if (!frameSource)
        frameSource = createFrameSource_Video(inputVideoName);

    // 跳过第一帧
    Mat frame;
    frameSource->nextFrame(frame);
    // 设置输入源
    superRes->setInput(frameSource);

    VideoWriter writer;

    for (int i = 0;; ++i)
    {
        Mat result;
        // 执行视频图像超分
        superRes->nextFrame(result);

        if (result.empty())
            break;

        imshow("Super Resolution", result);

        if (waitKey(1000) > 0)
            break;

        // 视频帧超分结果写到输出文件
        if (!outputVideoName.empty())
        {
            if (!writer.isOpened())
                writer.open(outputVideoName, VideoWriter::fourcc('X', 'V', 'I', 'D'), 25.0, result.size());
            writer << result;
        }
    }

    return 0;
}

二、视频超分源码

1、构造函数

视频图像超分的源码在opencv_contrib/modules/superres中，创建超分实例的函数位于btv_l1.cpp，其实是创建BTVL1()构造函数的智能指针：

Ptr<cv::superres::SuperResolution> cv::superres::createSuperResolution_BTVL1()
{
    return makePtr<BTVL1>();
}

2、超分入口代码

而nextFrame()是执行超分的函数，位于super_resolution.cpp：

void cv::superres::SuperResolution::nextFrame(OutputArray frame)
{
    isUmat_ = frame.isUMat() && cv::ocl::useOpenCL();

    if (firstCall_)
    {
        initImpl(frameSource_);
        firstCall_ = false;
    }

    processImpl(frameSource_, frame);
}

可以看到内部调用processImpl()函数来执行超分：

    void BTVL1::processImpl(Ptr<FrameSource>& frameSource, OutputArray _output)
    {
        if (outPos_ >= storePos_)
        {
            _output.release();
            return;
        }
        // 读取下一个视频帧
        readNextFrame(frameSource);
        // 处理视频帧
        if (procPos_ < storePos_)
        {
            ++procPos_;
            processFrame(procPos_);
        }
        ++outPos_;
        // 调用ocl_processImpl函数执行超分
        CV_OCL_RUN(isUmat_,
                   ocl_processImpl(frameSource, _output))

        const Mat& curOutput = at(outPos_, outputs_);

        if (_output.kind() < _InputArray::OPENGL_BUFFER || _output.isUMat())
            curOutput.convertTo(_output, CV_8U);
        else
        {
            curOutput.convertTo(finalOutput_, CV_8U);
            arrCopy(finalOutput_, _output);
        }
    }

3、光流检测运动矢量

接着看readNextFrame()的实现，调用光流检测算法来计算运动矢量：

void BTVL1::readNextFrame(Ptr<FrameSource>& frameSource)
    {
        frameSource->nextFrame(curFrame_);
        if (curFrame_.empty())
            return;

        ++storePos_;

        CV_OCL_RUN(isUmat_,
                   ocl_readNextFrame(frameSource))

        curFrame_.convertTo(at(storePos_, frames_), CV_32F);
        // 结合上一帧和当前帧计算运动矢量
        if (storePos_ > 0)
        {
            opticalFlow_->calc(prevFrame_, curFrame_, at(storePos_ - 1, forwardMotions_));
            opticalFlow_->calc(curFrame_, prevFrame_, at(storePos_, backwardMotions_));
        }

        curFrame_.copyTo(prevFrame_);
    }

4、处理超分

processFrame()函数负责处理视频帧：

void BTVL1::processFrame(int idx)
    {
        CV_OCL_RUN(isUmat_,
                   ocl_processFrame(idx))

        const int startIdx = std::max(idx - temporalAreaRadius_, 0);
        const int procIdx = idx;
        const int endIdx = std::min(startIdx + 2 * temporalAreaRadius_, storePos_);

        const int count = endIdx - startIdx + 1;

        srcFrames_.resize(count);
        srcForwardMotions_.resize(count);
        srcBackwardMotions_.resize(count);

        int baseIdx = -1;

        for (int i = startIdx, k = 0; i <= endIdx; ++i, ++k)
        {
            if (i == procIdx)
                baseIdx = k;

            srcFrames_[k] = at(i, frames_);

            if (i < endIdx)
                srcForwardMotions_[k] = at(i, forwardMotions_);
            if (i > startIdx)
                srcBackwardMotions_[k] = at(i, backwardMotions_);
        }
        // 根据前后方向的运动矢量来处理视频帧
        process(srcFrames_, at(idx, outputs_), srcForwardMotions_, srcBackwardMotions_, baseIdx);
    }
}

在计算得到前后方向的运动矢量后，调用BTVL1_Base基类的process()函数来处理：

    void BTVL1_Base::process(InputArrayOfArrays _src, OutputArray _dst, InputArrayOfArrays _forwardMotions,
                             InputArrayOfArrays _backwardMotions, int baseIdx)
    {
        CV_OCL_RUN(_src.isUMatVector() && _dst.isUMat() && _forwardMotions.isUMatVector() &&
                   _backwardMotions.isUMatVector(),
                   ocl_process(_src, _dst, _forwardMotions, _backwardMotions, baseIdx))

        std::vector<Mat> & src = *(std::vector<Mat> *)_src.getObj(),
                & forwardMotions = *(std::vector<Mat> *)_forwardMotions.getObj(),
                & backwardMotions = *(std::vector<Mat> *)_backwardMotions.getObj();

        // 更新运动模糊(高斯滤波)、 btv权重
        if (blurKernelSize_ != curBlurKernelSize_ || blurSigma_ != curBlurSigma_ || src[0].type() != curSrcType_)
        {
            //filter_ = createGaussianFilter(src[0].type(), Size(blurKernelSize_, blurKernelSize_), blurSigma_);
            curBlurKernelSize_ = blurKernelSize_;
            curBlurSigma_ = blurSigma_;
            curSrcType_ = src[0].type();
        }

        if (btvWeights_.empty() || btvKernelSize_ != curBtvKernelSize_ || alpha_ != curAlpha_)
        {
            calcBtvWeights(btvKernelSize_, alpha_, btvWeights_);
            curBtvKernelSize_ = btvKernelSize_;
            curAlpha_ = alpha_;
        }

        // 计算相对运动
        calcRelativeMotions(forwardMotions, backwardMotions, lowResForwardMotions_, lowResBackwardMotions_, baseIdx, src[0].size());
        // 对运动矢量进行放大
        upscaleMotions(lowResForwardMotions_, highResForwardMotions_, scale_);
        upscaleMotions(lowResBackwardMotions_, highResBackwardMotions_, scale_);

        forwardMaps_.resize(highResForwardMotions_.size());
        backwardMaps_.resize(highResForwardMotions_.size());
        for (size_t i = 0; i < highResForwardMotions_.size(); ++i)
            buildMotionMaps(highResForwardMotions_[i], highResBackwardMotions_[i], forwardMaps_[i], backwardMaps_[i]);


        const Size lowResSize = src[0].size();
        const Size highResSize(lowResSize.width * scale_, lowResSize.height * scale_);

        resize(src[baseIdx], highRes_, highResSize, 0, 0, INTER_CUBIC);

        diffTerm_.create(highResSize, highRes_.type());
        a_.create(highResSize, highRes_.type());
        b_.create(highResSize, highRes_.type());
        c_.create(lowResSize, highRes_.type());

        for (int i = 0; i < iterations_; ++i)
        {
            diffTerm_.setTo(Scalar::all(0));

            for (size_t k = 0; k < src.size(); ++k)
            {
                // a = M * Ih
                remap(highRes_, a_, backwardMaps_[k], noArray(), INTER_NEAREST);
                // 高斯模糊 b = HM * Ih
                GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
                // c = DHM * Ih
                resize(b_, c_, lowResSize, 0, 0, INTER_NEAREST);

                diffSign(src[k], c_, c_);

                // 超分运算 a = Dt * diff
                upscale(c_, a_, scale_);
                // b = HtDt * diff
                GaussianBlur(a_, b_, Size(blurKernelSize_, blurKernelSize_), blurSigma_);
                // a = MtHtDt * diff
                remap(b_, a_, forwardMaps_[k], noArray(), INTER_NEAREST);

                add(diffTerm_, a_, diffTerm_);
            }

            if (lambda_ > 0)
            {
                calcBtvRegularization(highRes_, regTerm_, btvKernelSize_, btvWeights_, ubtvWeights_);
                addWeighted(diffTerm_, 1.0, regTerm_, -lambda_, 0.0, diffTerm_);
            }

            addWeighted(highRes_, 1.0, diffTerm_, tau_, 0.0, highRes_);
        }

        Rect inner(btvKernelSize_, btvKernelSize_, highRes_.cols - 2 * btvKernelSize_, highRes_.rows - 2 * btvKernelSize_);
        highRes_(inner).copyTo(_dst);
    }

我们再看下upscaleMotions()函数的实现：

void upscaleMotions(InputArrayOfArrays _lowResMotions, OutputArrayOfArrays _highResMotions, int scale)
    {
        CV_OCL_RUN(_lowResMotions.isUMatVector() && _highResMotions.isUMatVector(),
                   ocl_upscaleMotions(_lowResMotions, _highResMotions, scale))

        std::vector<Mat> & lowResMotions = *(std::vector<Mat> *)_lowResMotions.getObj(),
                & highResMotions = *(std::vector<Mat> *)_highResMotions.getObj();

        highResMotions.resize(lowResMotions.size());

        for (size_t i = 0; i < lowResMotions.size(); ++i)
        {
            // 三次样条差值
            resize(lowResMotions[i], highResMotions[i], Size(), scale, scale, INTER_CUBIC);
            // 运动矢量与缩放因子相乘
            multiply(highResMotions[i], Scalar::all(scale), highResMotions[i]);
        }
    }

接着是upscale()超分函数的实现：

    void upscale(InputArray _src, OutputArray _dst, int scale)
    {
        int cn = _src.channels();
        CV_Assert( cn == 1 || cn == 3 || cn == 4 );

        CV_OCL_RUN(_dst.isUMat(),
                   ocl_upscale(_src, _dst, scale))

        typedef void (*func_t)(InputArray src, OutputArray dst, int scale);
        static const func_t funcs[] =
        {
            0, upscaleImpl<float>, 0, upscaleImpl<Point3f>, upscaleImpl<Point4f>
        };

        const func_t func = funcs[cn];
        CV_Assert(func != 0);
        func(_src, _dst, scale);
    }

最终是调用upscaleImpl()进行超分运算，属于模板函数：

    template <typename T>
    void upscaleImpl(InputArray _src, OutputArray _dst, int scale)
    {
        Mat src = _src.getMat();
        _dst.create(src.rows * scale, src.cols * scale, src.type());
        _dst.setTo(Scalar::all(0));
        Mat dst = _dst.getMat();

        for (int y = 0, Y = 0; y < src.rows; ++y, Y += scale)
        {
            const T * const srcRow = src.ptr<T>(y);
            T * const dstRow = dst.ptr<T>(Y);

            for (int x = 0, X = 0; x < src.cols; ++x, X += scale)
                dstRow[X] = srcRow[x];
        }
    }