LinkSVP例子之五：机器视觉基础演示，使用LinkIVE、NCNN、IVE进行人脸检测与追踪

版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。

本文链接： https://blog.csdn.net/weixin_45326556/article/details/100029823

背景

    参见LinkSVP例子之一背静介绍。

LinkSVP简介

    参见LinkSVP例子之一LinkSVP介绍。

FaceDetect人脸检测与追踪示例

该示例程序演示了如何综合使用LinkIVE、NCNN、IVE进行一个人脸检测与追踪的功能开发。

准备工作

• 参照用户手册搭建开发环境、编译3531D工程、配置网络启动参数。
• 将带有HDMI输出功能的设备(如摄像机、笔记本、机顶盒等)接入评估板的HDMI-A接口
• 将评估板的HDMI-OUT接显示器(支持1080P即可，程序默认输出1080P60)。
• 上电，进入/root/demo目录
• 运行FaceDetect程序

运行结果

完整工程

完整工程参见：https://gitee.com/LinkPi/LinkSVP/tree/master/FaceDetect

主要源代码

main.cpp

#include <QCoreApplication>
#include "Link.h"
#include "FaceDetect.h"

int main(int argc, char *argv[])
{
    QCoreApplication a(argc, argv);
    Link::init();

    LinkObject *vi=Link::create("InputVi");
    QVariantMap dataVi;
    dataVi["interface"]="HDMI-A";
    vi->start(dataVi);

    FaceDetect *FD=new FaceDetect();
    QVariantMap dataFD;
    dataFD["framerate"]=5;
    FD->start(dataFD);

    LinkObject *vo=Link::create("OutputVo");
    QVariantMap dataVo;
    dataVo["type"]="hdmi";
    vo->start(dataVo);

    vi->linkV(FD)->linkV(vo);

    return a.exec();
}

MTCNN.cpp

#include "MTCNN.h"

bool cmpScore(orderScore lsh, orderScore rsh){
    if(lsh.score<rsh.score)
        return true;
    else
        return false;
}

MTCNN::MTCNN(QObject *parent) : QObject(parent)
{
    Pnet.load_param("model/det1-opt.param");
    Pnet.load_model("model/det1-opt.bin");
    Rnet.load_param("model/det2-opt.param");
    Rnet.load_model("model/det2-opt.bin");
    Onet.load_param("model/det3-opt.param");
    Onet.load_model("model/det3-opt.bin");

}

void MTCNN::detect(IVEMem &rgb)
{
    mem=rgb;
    QMetaObject::invokeMethod(this,"doDetect",Qt::QueuedConnection);
}

void MTCNN::generateBbox(ncnn::Mat score, ncnn::Mat location, std::vector<Bbox>& boundingBox_, std::vector<orderScore>& bboxScore_, float scale){
    int stride = 2;
    int cellsize = 12;
    int count = 0;
    //score p
    float *p = score.channel(1);
    float *plocal = location.channel(0);
    Bbox bbox;
    orderScore order;
    for(int row=0;row<score.h;row++){
        for(int col=0;col<score.w;col++){
            if(*p>threshold[0]){
                bbox.score = *p;
                order.score = *p;
                order.oriOrder = count;
                bbox.x1 = round((stride*col+1)/scale);
                bbox.y1 = round((stride*row+1)/scale);
                bbox.x2 = round((stride*col+1+cellsize)/scale);
                bbox.y2 = round((stride*row+1+cellsize)/scale);
                bbox.exist = true;
                bbox.area = (bbox.x2 - bbox.x1)*(bbox.y2 - bbox.y1);
                for(int channel=0;channel<4;channel++)
                    bbox.regreCoord[channel]=location.channel(channel)[0];
                boundingBox_.push_back(bbox);
                bboxScore_.push_back(order);
                count++;
            }
            p++;
            plocal++;
        }
    }
}

void MTCNN::nms(std::vector<Bbox> &boundingBox_, std::vector<orderScore> &bboxScore_, const float overlap_threshold, string modelname){
    if(boundingBox_.empty()){
        return;
    }
    std::vector<int> heros;
    //sort the score
    sort(bboxScore_.begin(), bboxScore_.end(), cmpScore);

    int order = 0;
    float IOU = 0;
    float maxX = 0;
    float maxY = 0;
    float minX = 0;
    float minY = 0;
    while(bboxScore_.size()>0){
        order = bboxScore_.back().oriOrder;
        bboxScore_.pop_back();
        if(order<0)continue;
        if(boundingBox_.at(order).exist == false) continue;
        heros.push_back(order);
        boundingBox_.at(order).exist = false;//delete it

        for(int num=0;num<boundingBox_.size();num++){
            if(boundingBox_.at(num).exist){
                //the iou
                maxX = (boundingBox_.at(num).x1>boundingBox_.at(order).x1)?boundingBox_.at(num).x1:boundingBox_.at(order).x1;
                maxY = (boundingBox_.at(num).y1>boundingBox_.at(order).y1)?boundingBox_.at(num).y1:boundingBox_.at(order).y1;
                minX = (boundingBox_.at(num).x2<boundingBox_.at(order).x2)?boundingBox_.at(num).x2:boundingBox_.at(order).x2;
                minY = (boundingBox_.at(num).y2<boundingBox_.at(order).y2)?boundingBox_.at(num).y2:boundingBox_.at(order).y2;
                //maxX1 and maxY1 reuse
                maxX = ((minX-maxX+1)>0)?(minX-maxX+1):0;
                maxY = ((minY-maxY+1)>0)?(minY-maxY+1):0;
                //IOU reuse for the area of two bbox
                IOU = maxX * maxY;
                if(!modelname.compare("Union"))
                    IOU = IOU/(boundingBox_.at(num).area + boundingBox_.at(order).area - IOU);
                else if(!modelname.compare("Min")){
                    IOU = IOU/((boundingBox_.at(num).area<boundingBox_.at(order).area)?boundingBox_.at(num).area:boundingBox_.at(order).area);
                }
                if(IOU>overlap_threshold){
                    boundingBox_.at(num).exist=false;
                    for(vector<orderScore>::iterator it=bboxScore_.begin(); it!=bboxScore_.end();it++){
                        if((*it).oriOrder == num) {
                            (*it).oriOrder = -1;
                            break;
                        }
                    }
                }
            }
        }
    }
    for(int i=0;i<heros.size();i++)
        boundingBox_.at(heros.at(i)).exist = true;

}
void MTCNN::refineAndSquareBbox(vector<Bbox> &vecBbox, const int &height, const int &width){
    if(vecBbox.empty()){
        cout<<"Bbox is empty!!"<<endl;
        return;
    }

    float bbw=0, bbh=0, maxSide=0;
    float h = 0, w = 0;
    float x1=0, y1=0, x2=0, y2=0;
    for(vector<Bbox>::iterator it=vecBbox.begin(); it!=vecBbox.end();it++){
        if((*it).exist){
            bbw = (*it).x2 - (*it).x1 + 1;
            bbh = (*it).y2 - (*it).y1 + 1;
            x1 = (*it).x1 + (*it).regreCoord[0]*bbw;
            y1 = (*it).y1 + (*it).regreCoord[1]*bbh;
            x2 = (*it).x2 + (*it).regreCoord[2]*bbw;
            y2 = (*it).y2 + (*it).regreCoord[3]*bbh;

            w = x2 - x1 + 1;
            h = y2 - y1 + 1;

            maxSide = (h>w)?h:w;
            x1 = x1 + w*0.5 - maxSide*0.5;
            y1 = y1 + h*0.5 - maxSide*0.5;
            (*it).x2 = round(x1 + maxSide - 1);
            (*it).y2 = round(y1 + maxSide - 1);
            (*it).x1 = round(x1);
            (*it).y1 = round(y1);

            //boundary check
            if((*it).x1<0)(*it).x1=0;
            if((*it).y1<0)(*it).y1=0;
            if((*it).x2>width)(*it).x2 = width - 1;
            if((*it).y2>height)(*it).y2 = height - 1;

            it->area = (it->x2 - it->x1)*(it->y2 - it->y1);
        }
    }

}

void MTCNN::doDetect()
{

    std::vector<Bbox> finalBbox;

    ncnn::Mat ncnn_img = ncnn::Mat::from_pixels((uchar*)mem.data(), ncnn::Mat::PIXEL_BGR2RGB, 1920,1080);


    int cc=0;
    QVariantList list;
    QRect rt;
    detect_net(ncnn_img, finalBbox);
    for (vector<Bbox>::iterator it = finalBbox.begin(); it != finalBbox.end(); it++){
        if ((*it).exist)
        {
            cc++;
            rt=QRect((*it).x1, (*it).y1, ((*it).x2-(*it).x1), ((*it).y2-(*it).y1));
            list.append(rt);
        }
    }
    emit this->detectDone(list);
}

void MTCNN::detect_net(ncnn::Mat& img_, std::vector<Bbox>& finalBbox_){
    firstBbox_.clear();
    firstOrderScore_.clear();
    secondBbox_.clear();
    secondBboxScore_.clear();
    thirdBbox_.clear();
    thirdBboxScore_.clear();

    img = img_;
    img_w = img.w;
    img_h = img.h;

    img.substract_mean_normalize(mean_vals, norm_vals);


    float minl = img_w<img_h?img_w:img_h;
    int MIN_DET_SIZE = 12;
    int minsize = img_h/11;//90
    float m = (float)MIN_DET_SIZE/minsize;
    minl *= m;
    float factor = 0.5;//0.709
    int factor_count = 0;
    vector<float> scales_;
    while(minl>MIN_DET_SIZE){
        if(factor_count>0)m = m*factor;
        scales_.push_back(m);
        minl *= factor;
        factor_count++;
    }
    orderScore order;
    int count = 0;



    for (size_t i = 0; i < scales_.size(); i++) {
        int hs = (int)ceil(img_h*scales_[i]);
        int ws = (int)ceil(img_w*scales_[i]);
        ncnn::Mat in;

        resize_bilinear(img_, in, ws, hs);

        ncnn::Extractor ex = Pnet.create_extractor();
        ex.set_num_threads(2);
        ex.set_light_mode(true);
        ex.input("data", in);
        ncnn::Mat score_, location_;
        ex.extract("prob1", score_);
        ex.extract("conv4-2", location_);
        std::vector<Bbox> boundingBox_;
        std::vector<orderScore> bboxScore_;
        generateBbox(score_, location_, boundingBox_, bboxScore_, scales_[i]);
        nms(boundingBox_, bboxScore_, nms_threshold[0]);

        for(vector<Bbox>::iterator it=boundingBox_.begin(); it!=boundingBox_.end();it++){
            if((*it).exist){
                firstBbox_.push_back(*it);
                order.score = (*it).score;
                order.oriOrder = count;
                firstOrderScore_.push_back(order);
                count++;
            }
        }
        bboxScore_.clear();
        boundingBox_.clear();
    }
    //the first stage's nms
    if(count<1)return;
    nms(firstBbox_, firstOrderScore_, nms_threshold[0]);
    refineAndSquareBbox(firstBbox_, img_h, img_w);

    //second stage
    count = 0;
    for(vector<Bbox>::iterator it=firstBbox_.begin(); it!=firstBbox_.end();it++){
        if((*it).exist){
            ncnn::Mat tempIm;
            copy_cut_border(img, tempIm, (*it).y1, img_h-(*it).y2, (*it).x1, img_w-(*it).x2);
            ncnn::Mat in;

            resize_bilinear(tempIm, in, 24, 24);

            ncnn::Extractor ex = Rnet.create_extractor();
            ex.set_num_threads(2);
            ex.set_light_mode(true);
            ex.input("data", in);
            ncnn::Mat score, bbox;
            ex.extract("prob1", score);
            ex.extract("conv5-2", bbox);
            if((score[1])>threshold[1]){
                for(int channel=0;channel<4;channel++)
                    it->regreCoord[channel]=bbox[channel];
                it->area = (it->x2 - it->x1)*(it->y2 - it->y1);
                it->score = score[1];
                secondBbox_.push_back(*it);
                order.score = it->score;
                order.oriOrder = count++;
                secondBboxScore_.push_back(order);
            }
            else{
                (*it).exist=false;
            }
        }
    }

    if(count<1)return;
    nms(secondBbox_, secondBboxScore_, nms_threshold[1]);
    refineAndSquareBbox(secondBbox_, img_h, img_w);

    //third stage
    count = 0;
    for(vector<Bbox>::iterator it=secondBbox_.begin(); it!=secondBbox_.end();it++){
        if((*it).exist){
            ncnn::Mat tempIm;
            copy_cut_border(img, tempIm, (*it).y1, img_h-(*it).y2, (*it).x1, img_w-(*it).x2);
            ncnn::Mat in;

            resize_bilinear(tempIm, in, 48, 48);

            ncnn::Extractor ex = Onet.create_extractor();
            ex.set_num_threads(2);
            ex.set_light_mode(true);
            ex.input("data", in);
            ncnn::Mat score, bbox, keyPoint;
            ex.extract("prob1", score);
            ex.extract("conv6-2", bbox);
            ex.extract("conv6-3", keyPoint);

            if(score[1]>threshold[2]){
                for(int channel=0;channel<4;channel++)
                    it->regreCoord[channel]=bbox[channel];
                it->area = (it->x2 - it->x1)*(it->y2 - it->y1);
                it->score = score[1];
                for(int num=0;num<5;num++){
                    (it->ppoint)[num] = it->x1 + (it->x2 - it->x1)*keyPoint[num];
                    (it->ppoint)[num+5] = it->y1 + (it->y2 - it->y1)*keyPoint[num+5];
                }

                thirdBbox_.push_back(*it);
                order.score = it->score;
                order.oriOrder = count++;
                thirdBboxScore_.push_back(order);
            }
            else
                (*it).exist=false;
        }
    }


    if(count<1)return;
    refineAndSquareBbox(thirdBbox_, img_h, img_w);
    nms(thirdBbox_, thirdBboxScore_, nms_threshold[2], "Min");

    finalBbox_ = thirdBbox_;


}

FaceDetect.cpp

#include "FaceDetect.h"

FaceDetect::FaceDetect(QObject *parent) : LinkFrame(parent)
{
    data["framerate"]=5;
    data["width"]=1920;
    data["height"]=1080;
    mem["rgb"]=IVEMem(1920,1080,3,true);
    mem["rect"]=IVEMem(1920,1080,1,true);
    mem["mat"]=IVEMem(1920/8,1080/8,1,true);
    mem["mat2"]=IVEMem(1920/8,1080/8,1,true);
    trackers=NULL;
    connect(&detector,SIGNAL(detectDone(QVariantList)),this,SLOT(onDetect(QVariantList)));
}


void FaceDetect::oneFrame()
{
    int scale=8;
    static int ccc=-1;
    ccc=(ccc+1)%5;
    if(ccc==0)
    {

        IVE_CSC_CTRL_S ctrlCSC;
        ctrlCSC.enMode=IVE_CSC_MODE_PIC_BT709_YUV2RGB;
        HI_S32 ret=HI_MPI_IVE_CSC(&handle, mem["in"].toImage(IVE_IMAGE_TYPE_YUV420SP), mem["rgb"].toImage(IVE_IMAGE_TYPE_U8C3_PACKAGE), &ctrlCSC, HI_TRUE);
        if(ret!=HI_SUCCESS)
        {
            qDebug("HI_MPI_IVE_CSC failed %#x",ret);
        }
        copySmall(mem["in"],mem["mat"]);
        wait();
        detector.detect(mem["rgb"]);
    }

    {
        if(trackers==NULL)
            return;
        copySmall(mem["in"],mem["mat2"]);
        wait();
        mat2=Mat(1080/scale, 1920/scale, CV_8UC1, (void*)mem["mat2"].data());

        struct timeval tpstart,tpend;
        gettimeofday(&tpstart,NULL);
        trackers->update(mat2);

        gettimeofday(&tpend,NULL);

        copy(mem["in"],mem["rect"]);
        IVEMem UVin=mem["in"].toUV();
        IVEMem UVout=mem["out"].toUV();
        copy(UVin,UVout);
        wait();

        matout=Mat(1080, 1920, CV_8UC1, (void*)mem["rect"].data());

        QVariantList ret;
        for(int i=0;i<trackers->getObjects().size();i++)
        {
            Rect rt1=trackers->getObjects()[i];
            Rect rt2;
            rt2.x=rt1.x*scale;//-0.33*rt1.width*scale;
            rt2.y=rt1.y*scale;//-0.33*rt1.height*scale;
            rt2.width=rt1.width*scale;//+0.66*rt1.width*scale;
            rt2.height=rt1.height*scale;//+0.66*rt1.height*scale;
           rectangle( matout, rt2, Scalar( 255, 0, 0 ), 4, 1 );

           QVariantMap rect;
           rect["x"]=(double)rt2.x/1920.0;
           rect["y"]=(double)rt2.y/1080.0;
           rect["w"]=(double)rt2.width/1920.0;
           rect["h"]=(double)rt2.height/1080.0;

           ret.push_back(rect);
        }

        emit this->newEvent("FD",ret);

        mem["rect"].flush();
        copy(mem["rect"],mem["out"]);

        wait();

    }
}

void FaceDetect::drawRect(QRect rect)
{
    char *data=mem["rect"].data();
    for(int i=0;i<5;i++)
    {
        memset(data+1920*(rect.top()+i)+rect.left(),255,rect.width());
        memset(data+1920*(rect.bottom()-i)+rect.left(),255,rect.width());
    }

    for(int i=0;i<rect.height();i++)
    {
        memset(data+1920*(rect.top()+i)+rect.left(),255,5);
        memset(data+1920*(rect.top()+i)+rect.right()-5,255,5);
    }

}

void FaceDetect::onDetect(QVariantList list)
{
    int scale=8;
    if(trackers!=NULL)
    {
        delete trackers;
        trackers=NULL;
    }


    wait();
    mat=Mat(1080/scale, 1920/scale, CV_8UC1, (void*)mem["mat"].data());

    trackers=new MultiTracker();
    std::vector<Ptr<Tracker> > algorithms;
    vector<Rect2d> objects;
    for(int i=0;i<list.count();i++)
    {
        Rect rt;
        QRect rect=list[i].toRect();
        rt.x=rect.x()/scale;//+0.2*rect.width()/scale;
        rt.y=rect.y()/scale;//+0.2*rect.height()/scale;
        rt.width=rect.width()/scale;//-0.4*rect.width()/scale;
        rt.height=rect.height()/scale;//-0.4*rect.height()/scale;
        objects.push_back(rt);
        algorithms.push_back(TrackerKCF::create());

    }
    trackers->add(algorithms,mat,objects);
}