Ex4：Image Warping and Image Morphing

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/sinat_31790817/article/details/80278715

Ex4：Image Warping and Image Morphing

完整代码见github地址：https://github.com/linjiafengyang/ComputerVision

作业4

测试数据

普通A4打印纸，上面可能有手写笔记或者打印内容，但是拍照时可能角度不正。（参考Dataset2）。
输出：
已经矫正好的标准普通A4纸（长宽比为A4纸的比例），并裁掉无用的其他内容，只保留完整A4纸张。
实验数据集依旧使用作业3的数据集。

说明：尽量保留A4纸上的原有信息不变或减少图像信息（原图像内容不要做缩放，A4纸的面积基本不变），以利于后期的文字识别和处理。

测试结果

以下面两张图为例：

代码文件

在作业3的基础上，先求A4纸的四个顶点，再进行Warping为标准普通A4纸，最后裁剪。

ImageWarping.h文件：

#include "CImg.h"
#include <iostream>
#include <vector>
using namespace cimg_library;
using namespace std;

struct Point {
    double x, y;
    int cnt;
    Point(double _x, double _y, int _cnt): x(_x), y(_y), cnt(_cnt) {}
};
struct Line {
    double k, b;
    Line(double _k, double _b): k(_k), b(_b) {}
    int index;
    double distance;
    Line(int _index, double _distance): index(_index), distance(_distance) {}
};
class Hough {
private:
    CImg<float> grayImage; // 灰度图
    CImg<float> blurred_img; // 高斯滤波平滑得到的图
    CImg<float> houghspace; // 霍夫空间图
    CImg<float> hough_result; // 霍夫检测图
    CImg<float> A4; // a4纸结果图
    vector<Point> peaks; // 霍夫空间直线经过最多的点
    vector<Line> lines; // 直线
    vector<Point> intersections; // 直线交点

    double sigma;
    double gradient_threshold;
    double vote_threshold;
    double peak_dis;
    int x_min, x_max, y_min, y_max;
public:
    Hough(CImg<float> srcImg, double sigma, double gradient_threshold, double vote_threshold, double peak_dis);
    CImg<float> imageWarping(CImg<float> srcImg);
    CImg<float> RGBtoGray(const CImg<float>& srcImg); // 转灰度图
    CImg<float> initHoughSpace(); // 初始化霍夫空间
    void findPeaks(); // 投票算法
    void drawLines(); // 寻找并画出直线
    void drawIntersections(); // 寻找并画出直线交点
    vector<CImg<float> > computeTransformMatrix(CImg<float> a4); // 计算变换矩阵
    CImg<float> warping(CImg<float> srcImg); // image warping
};

ImageWarping.cpp文件：

#include "ImageWarping.h"

Hough::Hough(CImg<float> srcImg, double sigma, double gradient_threshold, double vot_threshold, double peak_dis) {
    this->hough_result = srcImg;
    this->sigma = sigma;
    this->gradient_threshold = gradient_threshold;
    this->vote_threshold = vot_threshold;
    this->peak_dis = peak_dis;
    this->x_min = 0;
    this->x_max = srcImg._width - 1; // 图像宽度
    this->y_min = 0;
    this->y_max = srcImg._height - 1; // 图像高度
}
CImg<float> Hough::imageWarping(CImg<float> srcImg) {
    this->grayImage = RGBtoGray(srcImg); // 转灰度图
    this->blurred_img = grayImage.get_blur(sigma); // 高斯滤波平滑
    this->houghspace = initHoughSpace(); // 初始化霍夫空间
    findPeaks(); // 找出霍夫空间中直线经过最多的点
    drawLines(); // 寻找并画出直线
    drawIntersections(); // 寻找并画出直线交点
    //hough_result.display();
    this->A4 = warping(srcImg); // image warping
    return A4;
}
// 转灰度图
CImg<float> Hough::RGBtoGray(const CImg<float>& srcImg) {
    CImg<float> grayImage = CImg<float>(srcImg._width, srcImg._height, 1, 1, 0);
    cimg_forXY(grayImage, x, y) {
        grayImage(x, y, 0) = (int)round((double)srcImg(x, y, 0, 0) * 0.299 + 
                                    (double)srcImg(x, y, 0, 1) * 0.587 + 
                                    (double)srcImg(x, y, 0, 2) * 0.114);
    }
    return grayImage;
}
// 初始化霍夫空间
CImg<float> Hough::initHoughSpace() {
    CImgList<float> gradient =  blurred_img.get_gradient("xy", 2);

    CImg<float> gradient_x = gradient[0]; // x方向上的梯度
    CImg<float> gradient_y = gradient[1]; // y方向上的梯度

    int maxp = (int)sqrt(grayImage._width*grayImage._width + grayImage._height*grayImage._height);
    CImg<float> hough_space(360, maxp, 1, 1, 0); // 初始化hough space

    cimg_forXY(grayImage, i, j) {
        double grad = sqrt(gradient_x(i, j)*gradient_x(i, j) + gradient_y(i, j)*gradient_y(i, j));
        if (grad > gradient_threshold) {
            grayImage(i, j) = grad;
            cimg_forX(hough_space, alpha) {
                double theta = ((double)alpha*cimg::PI) / 180;
                int p = (int)(i*cos(theta) + j*sin(theta));
                if (p >= 0 && p < maxp) {
                    hough_space(alpha, p)++; // 累加矩阵
                }
            }
        }
    }
    return hough_space;
}
// 投票算法找出霍夫空间中直线经过最多的点
void Hough::findPeaks() {
    peaks.clear();
    cimg_forXY(houghspace, theta, p) {
        if (houghspace(theta, p) > vote_threshold) {
            bool flag = true;
            double alpha = (double)theta*cimg::PI / 180;
            // y的范围
            const int y0 = ((double)p / (sin(alpha))) - double(x_min)*(1 / tan(alpha));
            const int y1 = ((double)p / (sin(alpha))) - double(x_max)*(1 / tan(alpha));
            // x的范围
            const int x0 = ((double)p / (cos(alpha))) - double(y_min)*(tan(alpha));
            const int x1 = ((double)p / (cos(alpha))) - double(y_max)*(tan(alpha));

            if (x0 >= x_min && x0 <= x_max || x1 >= x_min && x1 <= x_max ||
                y0 >= y_min && y0 <= y_max || y1 >= y_min && y1 <= y_max) {
                for (int i = 0; i < peaks.size(); i++) {
                    if (sqrt((peaks[i].x - theta)*(peaks[i].x - theta) 
                        + (peaks[i].y - p)*(peaks[i].y - p)) < peak_dis) {
                        flag = false;
                        if (peaks[i].cnt < houghspace(theta, p)) {
                            Point temp(theta, p, houghspace(theta, p));
                            peaks[i] = temp;
                        }
                    }
                }
                if (flag) {
                    Point temp(theta, p, houghspace(theta, p));
                    peaks.push_back(temp);
                }
            }
        }
    }
}
// 寻找并画出直线
void Hough::drawLines() {
    lines.clear();
    for (int i = 0; i < peaks.size(); i++) {
        double theta = double(peaks[i].x)*cimg::PI / 180;
        double k = -cos(theta) / sin(theta); // 直线斜率
        double b = double(peaks[i].y) / sin(theta);
        Line templine(k, b);
        lines.push_back(templine);
        cout << "Line " << i << ": y = " << k << "x + " << b << endl;
    }

    const double lines_color[] = { 255, 0, 0 };
    for (int i = 0; i < lines.size(); i++) {
        const int x0 = (double)(y_min - lines[i].b) / lines[i].k;
        const int x1 = (double)(y_max - lines[i].b) / lines[i].k;
        const int y0 = x_min*lines[i].k + lines[i].b;
        const int y1 = x_max*lines[i].k + lines[i].b;

        if (abs(lines[i].k) > 1) {
            hough_result.draw_line(x0, y_min, x1, y_max, lines_color);
        }
        else {
            hough_result.draw_line(x_min, y0, x_max, y1, lines_color);
        }
    }

}
bool sortByDistance(Line a, Line b) {
    return a.distance < b.distance;
}
// 寻找并画出直线交点
void Hough::drawIntersections() {
    vector<Point> tempIntersections;
    int k = 0;
    for (int i = 0; i < lines.size(); i++) {
        for (int j = i + 1; j < lines.size(); j++) {
            double k0 = lines[i].k, k1 = lines[j].k;
            double b0 = lines[i].b, b1 = lines[j].b;

            double x = (b1 - b0) / (k0 - k1);
            double y = (k0*b1 - k1*b0) / (k0 - k1);

            if (x >= 0 && x < grayImage._width && y >= 0 && y < grayImage._height) {
                Point tempPoint(x, y, 0);
                tempIntersections.push_back(tempPoint);
                cout << "Intersection " << k++ << ": x = " << x << ", y = " << y << endl;
            }
        }
    }

    const double intersections_color[] = { 255, 0, 0 };
    vector<Line> sort_line;
    for (int i = 0; i < tempIntersections.size(); i++) {
        hough_result.draw_circle(tempIntersections[i].x, tempIntersections[i].y, 50, intersections_color);

        double temp_x = tempIntersections[i].x - tempIntersections[0].x;
        double temp_y = tempIntersections[i].y - tempIntersections[0].y;
        double dis = sqrt(temp_x*temp_x + temp_y*temp_y);
        Line temp(i, dis);
        sort_line.push_back(temp);
    }
    sort(sort_line.begin(), sort_line.end(), sortByDistance);
    intersections.clear();
    for (int i = 0; i < sort_line.size(); i++) {
        intersections.push_back(tempIntersections[sort_line[i].index]); // 四个角点
    }
}
// 计算变换矩阵
vector<CImg<float> > Hough::computeTransformMatrix(CImg<float> a4) {
    vector<Point> destRectPoints;
    Point tempPoint1(0, 0, 0);
    Point tempPoint2(a4._width - 1, 0, 0);
    Point tempPoint3(0, a4._height - 1, 0);
    Point tempPoint4(a4._width - 1, a4._height - 1, 0);
    destRectPoints.push_back(tempPoint1); destRectPoints.push_back(tempPoint2);
    destRectPoints.push_back(tempPoint3); destRectPoints.push_back(tempPoint4);

    CImg<float> y1(1, 3, 1, 1, 0), y2(1, 3, 1, 1, 0), y3(1, 3, 1, 1, 0), y4(1, 3, 1, 1, 0);
    CImg<float> c1(1, 3, 1, 1, 0), c2(1, 3, 1, 1, 0), c3(1, 3, 1, 1, 0), c4(1, 3, 1, 1, 0);
    CImg<float> A1(3, 3, 1, 1, 1), A2(3, 3, 1, 1, 1);

    for (int i = 0; i < 3; i++) {
        A1(0, i) = destRectPoints[i].x; A1(1, i) = destRectPoints[i].y;
        A2(0, i) = destRectPoints[3-i].x; A2(1, i) = destRectPoints[3-i].y;

        y1(0, i) = intersections[i].x; y2(0, i) = intersections[i].y;
        y3(0, i) = intersections[3-i].x; y4(0, i) = intersections[3-i].y;
    }
    c1 = y1.solve(A1); c2 = y2.solve(A1);
    c3 = y3.solve(A2); c4 = y4.solve(A2);

    CImg<float> temptransform1(3, 3, 1, 1, 0), temptransform2(3, 3, 1, 1, 0);
    for (int i = 0; i < 3; i++) {
        temptransform1(i, 0) = c1(0, i);
        temptransform1(i, 1) = c2(0, i);

        temptransform2(i, 0) = c3(0, i);
        temptransform2(i, 1) = c4(0, i);
    }
    temptransform1(0, 2) = 0; temptransform1(1, 2) = 0; temptransform1(2, 2) = 1;
    temptransform2(0, 2) = 0; temptransform2(1, 2) = 0; temptransform2(2, 2) = 1;
    vector<CImg<float> > temptransform;
    temptransform.push_back(temptransform1);
    temptransform.push_back(temptransform2);
    return temptransform;
}
// image warping
CImg<float> Hough::warping(CImg<float> srcImg) {
    // 当分辨率是150像素/英寸时，A4纸像素长宽分别是1754×1240
    // 当分辨率是300像素/英寸时，A4纸像素长宽分别是3508×2479
    CImg<float> a4(1240, 1754, 1, 3, 0);
    //CImg<float> a4(2479, 3508, 1, 3, 0);
    vector<CImg<float> > transform;
    transform = computeTransformMatrix(a4); // 计算变换矩阵

    CImg<float> y(1, 2, 1, 1, 0);
    CImg<float> c(1, 2, 1, 1, 0);
    CImg<float> A(2, 2, 1, 1, 1);
    A(0, 0) = 0;
    A(0, 1) = a4._width - 1;
    y(0, 0) = a4._height - 1;
    y(0, 1) = 0;
    c = y.solve(A);

    CImg<float> temp1(1, 3, 1, 1, 1), temp2(1, 3, 1, 1, 1);
    cimg_forXY(a4, i, j) {
        temp1(0, 0) = i;
        temp1(0, 1) = j;

        double inner_procuct = i * c(0, 0) - j + c(0, 1);
        temp2 = inner_procuct >= 0 ? transform[0] * temp1 : transform[1] * temp1;
        temp2(0, 0) = temp2(0, 0) < 0 ? 0 : (temp2(0, 0) > x_max ? x_max : temp2(0, 0));
        temp2(0, 1) = temp2(0, 1) < 0 ? 0 : (temp2(0, 1) > y_max ? y_max : temp2(0, 1));

        a4(i, j, 0) = srcImg(temp2(0, 0), temp2(0, 1), 0);
        a4(i, j, 1) = srcImg(temp2(0, 0), temp2(0, 1), 1);
        a4(i, j, 2) = srcImg(temp2(0, 0), temp2(0, 1), 2);
    }
    return a4;
}

main.cpp文件：

#include "ImageWarping.cpp"
#include <time.h>
#include <iostream>
int main() {
    CImg<float> src("Dataset/6.bmp");
    // 4.bmp需要将sigma设为5.5f
    // args: src sigma gradient_threshold vote_threshold peak_dis
    Hough hough(src, 10.5f, 30, 1000, 60);
    CImg<float> result = hough.imageWarping(src);
    result.display();
    result.save("result/6.bmp");
    return 0;
}

附加题

扫描二维码关注公众号，回复： 4040038 查看本文章

根据Image Morphing方法完成中间11帧的差值，得到一个Image Morphing的动画视频。

测试结果

见github地址：https://github.com/linjiafengyang/ComputerVision

代码文件

ImageMorphing.h文件：

#include <iostream>
#include "CImg.h"
#include <vector>
#include <string>

using namespace std;
using namespace cimg_library;

struct Point {
    float x, y;
    Point(float _x, float _y): x(_x), y(_y) {}
};
struct Triangle {
    Point p1, p2, p3;
    Triangle(Point _p1, Point _p2, Point _p3): p1(_p1), p2(_p2), p3(_p3) {}
};
class ImageMorphing {
private:
    vector<Point> src_points;
    vector<Point> dst_points;
    vector<vector<Point> > mid_points;

    vector<vector<int> > index;
    vector<Triangle> src_triangle_list;
    vector<Triangle> dst_triangle_list;
    vector<vector<Triangle> > mid_triangle_list;

    CImg<float> src;
    CImg<float> dst;
    CImgList<float> result;

    int frame_cnt;
public:
    ImageMorphing(CImg<float> src, CImg<float> dst, vector<Point> src_points, vector<Point> dst_points, 
                    vector<vector<int> > index, int frame_cnt);
    void setSrcTriangleList();
    void setDstTriangleList();
    void setMidTriangleList();
    CImg<float> computeTransformMatrix(Triangle before, Triangle after); // 计算变换矩阵
    bool isInTriangle(Point P, Triangle tri); // 重心法判断一个点是否在三角形内
    CImgList<float> morphing();
};

ImageMorphing.cpp文件：

#include "ImageMorphing.h"

ImageMorphing::ImageMorphing(CImg<float> src, CImg<float> dst, vector<Point> src_points, 
                vector<Point> dst_points, vector<vector<int>> index, int frame_cnt) {
    this->src = src;
    this->dst = dst;
    this->src_points = src_points;
    this->dst_points = dst_points;
    this->index = index;
    this->frame_cnt = frame_cnt;
    setSrcTriangleList();
    setDstTriangleList();
    setMidTriangleList();
}
// src图的三角形
void ImageMorphing::setSrcTriangleList() {
    for (int i = 0; i < index.size(); i++) {
        Triangle src_triangle(src_points[index[i][0]], src_points[index[i][1]], src_points[index[i][2]]);
        src_triangle_list.push_back(src_triangle);
    }
}
// dst图的三角形
void ImageMorphing::setDstTriangleList() {
    for (int i = 0; i < index.size(); i++) {
        Triangle dst_triangle(dst_points[index[i][0]], dst_points[index[i][1]], dst_points[index[i][2]]);
        dst_triangle_list.push_back(dst_triangle);
    }
}
// morphing过程中每一帧的三角形
void ImageMorphing::setMidTriangleList() {
    for (int i = 0; i < frame_cnt; i++) {
        vector<Point> temp_mid_points;
        for (int j = 0; j < src_points.size(); j++) {
            float mid_x = float(src_points[j].x) + float(i+1)/(frame_cnt+1) * (dst_points[j].x - src_points[j].x);
            float mid_y = float(src_points[j].y) + float(i+1)/(frame_cnt+1) * (dst_points[j].y - src_points[j].y);
            Point mid_point(mid_x, mid_y);
            temp_mid_points.push_back(mid_point);
        }
        mid_points.push_back(temp_mid_points);
    }

    for (int i = 0; i < frame_cnt; i++) {
        vector<Triangle> temp_mid_triangles;
        for (int j = 0; j < index.size(); j++) {
            Triangle mid_triangle(mid_points[i][index[j][0]], mid_points[i][index[j][1]], mid_points[i][index[j][2]]);
            temp_mid_triangles.push_back(mid_triangle);
        }
        mid_triangle_list.push_back(temp_mid_triangles);
    }
}
// 重心法判断一个点是否在三角形内
bool ImageMorphing::isInTriangle(Point P, Triangle tri) {
    float x0 = tri.p3.x - tri.p1.x, y0 = tri.p3.y - tri.p1.y;
    float x1 = tri.p2.x - tri.p1.x, y1 = tri.p2.y - tri.p1.y;
    float x2 = P.x - tri.p1.x, y2 = P.y - tri.p1.y;

    float temp_00 = x0 * x0 + y0 * y0;
    float temp_01 = x0 * x1 + y0 * y1;
    float temp_02 = x0 * x2 + y0 * y2;
    float temp_11 = x1 * x1 + y1 * y1;
    float temp_12 = x1 * x2 + y1 * y2;

    float u = float(temp_11 * temp_02 - temp_01 * temp_12) / (float)(temp_00 * temp_11 - temp_01 * temp_01);
    float v = float(temp_00 * temp_12 - temp_01 * temp_02) / (float)(temp_00 * temp_11 - temp_01 * temp_01);
    if (u + v <= 1 && u >= 0 && v >= 0) return true;
    return false;
}
// 计算变换矩阵
CImg<float> ImageMorphing::computeTransformMatrix(Triangle before, Triangle after) {
    CImg<float> A(3, 3, 1, 1, 1);
    CImg<float> y1(1, 3, 1, 1, 0), y2(1, 3, 1, 1, 0);
    CImg<float> c1(1, 3, 1, 1, 0), c2(1, 3, 1, 1, 0);

    A(0, 0) = before.p1.x; A(1, 0) = before.p1.y;
    A(0, 1) = before.p2.x; A(1, 1) = before.p2.y;
    A(0, 2) = before.p3.x; A(1, 2) = before.p3.y;

    y1(0, 0) = after.p1.x; y2(0, 0) = after.p1.y;
    y1(0, 1) = after.p2.x; y2(0, 1) = after.p2.y;
    y1(0, 2) = after.p3.x; y2(0, 2) = after.p3.y;

    c1 = y1.solve(A);
    c2 = y2.solve(A);

    CImg<float> transform(3, 3, 1, 1, 0);
    for (int i = 0; i < 3; i++) {
        transform(i, 0) = c1(0, i);
        transform(i, 1) = c2(0, i);
    }
    transform(2, 2) = 1;
    return transform;
}
// 对每一帧进行morphing
CImgList<float> ImageMorphing::morphing() {
    int size = mid_triangle_list[0].size();

    result.push_back(src); // 放入第一帧
    for (int k = 0; k < frame_cnt; k++) {
        CImg<float> middle(dst._width, dst._height, 1, 3, 1);
        cimg_forXY(middle, i, j) {
            CImg<float> x(1, 3, 1, 1, 1), y1(1, 3, 1, 1, 1), y2(1, 3, 1, 1, 1);
            for (int m = 0; m < size; m++) {
                Point p(i, j);
                if (isInTriangle(p, mid_triangle_list[k][m])) {
                    x(0, 0) = i;
                    x(0, 1) = j;
                    // middle image到src image的变换
                    CImg<float> transform1 = computeTransformMatrix(mid_triangle_list[k][m], src_triangle_list[m]);
                    y1 = transform1 * x;
                    // middle image到dst image的变换
                    CImg<float> transform2 = computeTransformMatrix(mid_triangle_list[k][m], dst_triangle_list[m]);
                    y2 = transform2 * x;
                    // src和dst组合得到middle
                    float a = float(k+1)/(frame_cnt+1);
                    middle(i, j, 0) = (1 - a) * src(y1(0, 0), y1(0, 1), 0) + a * dst(y2(0, 0), y2(0, 1), 0);
                    middle(i, j, 1) = (1 - a) * src(y1(0, 0), y1(0, 1), 1) + a * dst(y2(0, 0), y2(0, 1), 1);
                    middle(i, j, 2) = (1 - a) * src(y1(0, 0), y1(0, 1), 2) + a * dst(y2(0, 0), y2(0, 1), 2);
                    break;
                }
            }
        }
        result.push_back(middle); // 放入中间morphing的每一帧
    }
    result.push_back(dst); // 放入最后一帧
    return result;
}

main.cpp文件：

#include "ImageMorphing.cpp"
#include <fstream>
#include <sstream>
#include <string>
int main() {
    CImg<float> src, dst;
    src.load_bmp("dataset/1.bmp");
    dst.load_bmp("dataset/2.bmp");

    // read source points
    vector<Point> src_points;
    ifstream infile_src_points;
    infile_src_points.open("points/src_points.txt");
    if (infile_src_points.is_open()) {
        string s;
        int x, y;
        while (getline(infile_src_points, s)) {
            stringstream ss(s);
            ss >> x >> y;
            Point p(x, y);
            src_points.push_back(p);
        }
    }
    else {
        cout << "Can't open the file src_points.txt" << endl;
        return 0;
    }

    // read destination points
    vector<Point> dst_points;
    ifstream infile_dst_points;
    infile_dst_points.open("points/dst_points.txt");
    if (infile_dst_points.is_open()) {
        string s;
        int x, y;
        while (getline(infile_dst_points, s)) {
            stringstream ss(s);
            ss >> x >> y;
            Point p(x, y);
            dst_points.push_back(p);
        }
    }
    else {
        cout << "Can't open the file dst_points.txt" << endl;
        return 0;
    }

    // read the indexs of triangles points
    vector<vector<int> > index;
    ifstream infile_tri;
    infile_tri.open("points/triangle.txt");
    if (infile_tri.is_open()) {
        string s;
        int index1, index2, index3;
        while (getline(infile_tri, s)) {
            vector<int> tempindex;
            stringstream ss(s);
            ss >> index1 >> index2 >> index3;
            tempindex.push_back(index1 - 1);
            tempindex.push_back(index2 - 1);
            tempindex.push_back(index3 - 1);
            index.push_back(tempindex);
        }
    }
    else {
        cout << "Can't open the file triangle.txt" << endl;
        return 0;
    }

    ImageMorphing imageMorphing(src, dst, src_points, dst_points, index, 11); // 11帧
    CImgList<float> result = imageMorphing.morphing();
    string dir_name;
    for (int i = 0; i < result.size(); i++) {
        dir_name = "result/";
        string s = to_string(i+1);
        s += ".bmp";
        dir_name += s;
        result[i].save_bmp(dir_name.c_str()); // Save every frame
    }
    return 0;
}