5. OpenCV image stitching

I. Introduction

   Image stitching (Image Stitching) is a technology that uses real-world images to form a panoramic space. It stitches multiple images into a large-scale image or a 360° panorama, which can be regarded as a special case of scene reconstruction. Planar homography for association. Image stitching has great applications in machine vision fields such as motion detection and tracking, augmented reality, resolution enhancement, video compression, and image stabilization.
  The output of image stitching is the union of the two input images .

1. Feature Extraction : Detect feature points in the input image.
2. Image Registration : It establishes the set correspondence between images so that they can be transformed, compared and analyzed in a common frame of reference.
3. Warping : Reproject the image of one of the images and place the image on a larger canvas.
4. Image fusion (Blending) : By changing the gray level of the image near the boundary, these gaps are removed, and a blended image is created to achieve a smooth transition between images. Blending Modes are used to blend two layers together.

2. Implementation method

Image stitching based on SURF

  Using the SIFT algorithm to achieve image stitching is a very common method, but because SIFT has a large amount of calculation, it is no longer applicable in occasions with high speed requirements. Therefore, its improved method SURF has a significant improvement in speed (the speed is 3 times that of SIFT), so it still has a lot to do in the field of image stitching. Although the accuracy and stability of SURF are not as good as SIFT, its comprehensive ability is still superior. The main steps of stitching are described in detail below.

1. Feature point extraction and matching

    //创建SURF对象
    //create 参数 海森矩阵阈值
    Ptr<SURF> surf;
    surf = SURF::create(800);

    //暴力匹配器
    BFMatcher matcher;

    vector<KeyPoint> key1, key2;
    Mat c, d;

    //寻找特征点
    surf->detectAndCompute(left, Mat(), key2, d);
    surf->detectAndCompute(right, Mat(), key1, c);

    //特征点对比 保存
    vector<DMatch>matches;

    //使用暴力匹配器匹配特征点 保存
    matcher.match(d, c, matches);

    //排序 从小到大
    sort(matches.begin(), matches.end());

    //保留最优的特征点收集
    vector<DMatch>good_matches;

    int ptrPoint = std::min(50, (int)(matches.size()*0.15));

    for(int i=0; i<ptrPoint; i++)
        good_matches.push_back(matches[i]);

    //最佳匹配的特征点连成一线
    Mat outimg;

    drawMatches(left, key2, right, key1, good_matches, outimg,
                Scalar::all(-1), Scalar::all(-1),
                vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);

    imshow("outimg", outimg);

2. Image registration

  In this way, the matching point sets of the two images to be stitched are obtained. Next, the image registration is performed, that is, the two images are converted to the same coordinates. Here, the findHomography function needs to be used to obtain the transformation matrix.
  However, it should be noted that the point set used by the findHomography function is of Point2f type, so it is necessary to process the point set good_matches just obtained to convert it into a point set of Point2f type.

    //特征点配准
    vector<Point2f>imagepoint1, imagepoint2;
    for(int i=0; i<good_matches.size(); i++)
    {
    
    
        imagepoint1.push_back(key1[good_matches[i].trainIdx].pt);
        imagepoint2.push_back(key2[good_matches[i].queryIdx].pt);
    }
   

  After the above operations, use imagepoint1 and imagepoint2 to find the transformation matrix and realize image registration.
  It is worth noting that CV_RANSAC is selected in the parameters of the findHomography function. Use the RANSAC algorithm to continue to screen reliable matching points, which makes the matching point solution more accurate.

//透视转换
    Mat homo = findHomography(imagepoint1, imagepoint2, CV_RANSAC);

    imshow("homo", homo);

    //右图四个顶点坐标转换计算
    CalcCorners(homo, right);

    Mat imageTransform;
    warpPerspective(right, imageTransform, homo,
                    Size(MAX(corners.right_top.x, corners.right_bottom.x), left.rows));
    imshow("imageTransform", imageTransform);

3. Image copy

The idea of ​​copying is very simple, just copy the left image directly to the registration image.

    int dst_width = imageTransform.cols;
    int dst_height = imageTransform.rows;

    Mat dst(dst_height, dst_width, CV_8UC3);
    dst.setTo(0);

    imageTransform.copyTo(dst(Rect(0, 0, imageTransform.cols, imageTransform.rows)));
    left.copyTo(dst(Rect(0, 0, left.cols, left.rows)));

4. Image fusion (de-crack processing)

    OptimizeSeam(left, imageTransform, dst);
    imshow("dst", dst);

    waitKey(0);

//优化两图的连接处，使得拼接自然
void OptimizeSeam(Mat& img1, Mat& trans, Mat& dst)
{
    
    
    int start = MIN(corners.left_top.x, corners.left_bottom.x);//开始位置，即重叠区域的左边界

    double processWidth = img1.cols - start;//重叠区域的宽度
    int rows = dst.rows;
    int cols = img1.cols; //注意，是列数*通道数
    double alpha = 1;//img1中像素的权重
    for (int i = 0; i < rows; i++)
    {
    
    
        uchar* p = img1.ptr<uchar>(i);  //获取第i行的首地址
        uchar* t = trans.ptr<uchar>(i);
        uchar* d = dst.ptr<uchar>(i);
        for (int j = start; j < cols; j++)
        {
    
    
            //如果遇到图像trans中无像素的黑点，则完全拷贝img1中的数据
            if (t[j * 3] == 0 && t[j * 3 + 1] == 0 && t[j * 3 + 2] == 0)
            {
    
    
                alpha = 1;
            }
            else
            {
    
    
                //img1中像素的权重，与当前处理点距重叠区域左边界的距离成正比，实验证明，这种方法确实好
                alpha = (processWidth - (j - start)) / processWidth;
            }
            d[j * 3] = p[j * 3] * alpha + t[j * 3] * (1 - alpha);
            d[j * 3 + 1] = p[j * 3 + 1] * alpha + t[j * 3 + 1] * (1 - alpha);
            d[j * 3 + 2] = p[j * 3 + 2] * alpha + t[j * 3 + 2] * (1 - alpha);
        }
    }
}

3. Complete code

#include <iostream>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/xfeatures2d.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgproc.hpp>

using namespace std;
using namespace cv;
using namespace cv::xfeatures2d;

typedef struct
{
    
    
    Point2f left_top;
    Point2f left_bottom;
    Point2f right_top;
    Point2f right_bottom;
}four_corners_t;

four_corners_t corners;

void CalcCorners(const Mat& H, const Mat& src)
{
    
    
    double v2[] = {
    
     0, 0, 1 };//左上角
    double v1[3];//变换后的坐标值
    Mat V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
    Mat V1 = Mat(3, 1, CV_64FC1, v1);  //列向量

    V1 = H * V2;
    //左上角(0,0,1)
    cout << "V2: " << V2 << endl;
    cout << "V1: " << V1 << endl;
    corners.left_top.x = v1[0] / v1[2];
    corners.left_top.y = v1[1] / v1[2];

    //左下角(0,src.rows,1)
    v2[0] = 0;
    v2[1] = src.rows;
    v2[2] = 1;
    V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
    V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
    V1 = H * V2;
    corners.left_bottom.x = v1[0] / v1[2];
    corners.left_bottom.y = v1[1] / v1[2];

    //右上角(src.cols,0,1)
    v2[0] = src.cols;
    v2[1] = 0;
    v2[2] = 1;
    V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
    V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
    V1 = H * V2;
    corners.right_top.x = v1[0] / v1[2];
    corners.right_top.y = v1[1] / v1[2];

    //右下角(src.cols,src.rows,1)
    v2[0] = src.cols;
    v2[1] = src.rows;
    v2[2] = 1;
    V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
    V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
    V1 = H * V2;
    corners.right_bottom.x = v1[0] / v1[2];
    corners.right_bottom.y = v1[1] / v1[2];

}

//优化两图的连接处，使得拼接自然
void OptimizeSeam(Mat& img1, Mat& trans, Mat& dst)
{
    
    
    int start = MIN(corners.left_top.x, corners.left_bottom.x);//开始位置，即重叠区域的左边界

    double processWidth = img1.cols - start;//重叠区域的宽度
    int rows = dst.rows;
    int cols = img1.cols; //注意，是列数*通道数
    double alpha = 1;//img1中像素的权重
    for (int i = 0; i < rows; i++)
    {
    
    
        uchar* p = img1.ptr<uchar>(i);  //获取第i行的首地址
        uchar* t = trans.ptr<uchar>(i);
        uchar* d = dst.ptr<uchar>(i);
        for (int j = start; j < cols; j++)
        {
    
    
            //如果遇到图像trans中无像素的黑点，则完全拷贝img1中的数据
            if (t[j * 3] == 0 && t[j * 3 + 1] == 0 && t[j * 3 + 2] == 0)
            {
    
    
                alpha = 1;
            }
            else
            {
    
    
                //img1中像素的权重，与当前处理点距重叠区域左边界的距离成正比，实验证明，这种方法确实好
                alpha = (processWidth - (j - start)) / processWidth;
            }
            d[j * 3] = p[j * 3] * alpha + t[j * 3] * (1 - alpha);
            d[j * 3 + 1] = p[j * 3 + 1] * alpha + t[j * 3 + 1] * (1 - alpha);
            d[j * 3 + 2] = p[j * 3 + 2] * alpha + t[j * 3 + 2] * (1 - alpha);
        }
    }
}

//计算配准图的四个顶点坐标
int main()
{
    
    
    Mat left = imread("A.jpg");
    Mat right =imread("B.jpg");

    imshow("left", left);
    imshow("right", right);

    //1.特征点提取和匹配
    //创建SURF对象
    //create 参数 海森矩阵阈值
    Ptr<SURF> surf;
    surf = SURF::create(800);

    //暴力匹配器
    BFMatcher matcher;

    vector<KeyPoint> key1, key2;
    Mat c, d;

    //寻找特征点
    surf->detectAndCompute(left, Mat(), key2, d);
    surf->detectAndCompute(right, Mat(), key1, c);

    //特征点对比 保存
    vector<DMatch>matches;

    //使用暴力匹配器匹配特征点 保存
    matcher.match(d, c, matches);

    //排序 从小到大
    sort(matches.begin(), matches.end());

    //保留最优的特征点收集
    vector<DMatch>good_matches;

    int ptrPoint = std::min(50, (int)(matches.size()*0.15));

    for(int i=0; i<ptrPoint; i++)
        good_matches.push_back(matches[i]);

    //最佳匹配的特征点连成一线
    Mat outimg;

    drawMatches(left, key2, right, key1, good_matches, outimg,
                Scalar::all(-1), Scalar::all(-1),
                vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);

    imshow("outimg", outimg);

    //2.图像配准
    //特征点配准
    vector<Point2f>imagepoint1, imagepoint2;
    for(int i=0; i<good_matches.size(); i++)
    {
    
    
        imagepoint1.push_back(key1[good_matches[i].trainIdx].pt);
        imagepoint2.push_back(key2[good_matches[i].queryIdx].pt);
    }

    //透视转换
    Mat homo = findHomography(imagepoint1, imagepoint2, CV_RANSAC);

    imshow("homo", homo);

    //右图四个顶点坐标转换计算
    CalcCorners(homo, right);

    Mat imageTransform;
    warpPerspective(right, imageTransform, homo,
                    Size(MAX(corners.right_top.x, corners.right_bottom.x), left.rows));
    imshow("imageTransform", imageTransform);

    
    //3.图像拷贝
    int dst_width = imageTransform.cols;
    int dst_height = imageTransform.rows;

    Mat dst(dst_height, dst_width, CV_8UC3);
    dst.setTo(0);

    imageTransform.copyTo(dst(Rect(0, 0, imageTransform.cols, imageTransform.rows)));
    left.copyTo(dst(Rect(0, 0, left.cols, left.rows)));

    //4.优化拼接最终结果图，去除黑边
    OptimizeSeam(left, imageTransform, dst);
    imshow("dst", dst);

    waitKey(0);

    return 0;
}