python-opencv epipolar geometry StereoRectify

How to use stereoRectify function correctly in OpenCV

Function introduction

    用于双目相机的立体校正环节中，这里只谈谈这个函数怎么使用，参数具体指哪些

Function parameters

    随便去网上一搜或者看官方手册就能得到参数信息，但是！！相对关系非常容易出错！！

    这里详细解释一下这些参数究竟怎么用

void stereoRectify(InputArray cameraMatrix1, InputArray distCoeffs1, 
　　　　　　　　　　　InputArray cameraMatrix2,InputArray distCoeffs2, Size imageSize, 
　　　　　　　　　　　InputArray R, InputArray T,OutputArray R1, OutputArray R2, OutputArray P1, 
　　　　　　　　　　　OutputArray P2, OutputArray Q, int flags=CALIB_ZERO_DISPARITY, double alpha=-1, 
　　　　　　　　　　　Size newImageSize=Size(), Rect* validPixROI1=0, Rect* validPixROI2=0 )

cameraMatrix1-第一个摄像机的摄像机矩阵，即左相机相机内参矩阵，矩阵第三行格式应该为 0 0 1
distCoeffs1-第一个摄像机的畸变向量
cameraMatrix2-第一个摄像机的摄像机矩阵，即右相机相机内参矩阵，矩阵第三行格式应该为 0 0 1
distCoeffs2-第二个摄像机的畸变向量
imageSize-图像大小
R- 相机之间的旋转矩阵，这里R的意义是：相机1通过变换R到达相机2的位姿
T-  左相机到右相机的平移矩阵
R1-输出矩阵，第一个摄像机的校正变换矩阵（旋转变换）
R2-输出矩阵，第二个摄像机的校正变换矩阵（旋转矩阵）
P1-输出矩阵，第一个摄像机在新坐标系下的投影矩阵
P2-输出矩阵，第二个摄像机在想坐标系下的投影矩阵
Q-4*4的深度差异映射矩阵
flags-可选的标志有两种零或者 CV_CALIB_ZERO_DISPARITY ,如果设置 CV_CALIB_ZERO_DISPARITY 的话，该函数会让两幅校正后的图像的主点有相同的像素坐标。否则该函数会水平或垂直的移动图像，以使得其有用的范围最大
alpha-拉伸参数。如果设置为负或忽略，将不进行拉伸。如果设置为0，那么校正后图像只有有效的部分会被显示（没有黑色的部分），如果设置为1，那么就会显示整个图像。设置为0~1之间的某个值，其效果也居于两者之间。
newImageSize-校正后的图像分辨率，默认为原分辨率大小。
validPixROI1-可选的输出参数，Rect型数据。其内部的所有像素都有效
validPixROI2-可选的输出参数，Rect型数据。其内部的所有像素都有效

Opencv performs dual-objective determination and epipolar correction python code

double goal setting

Reference blog OpenCV camera calibration process
[OpenCV Practical] 38 Camera calibration based on OpenCV
opencv stereo calibration function stereoCalibrate()

Mainly used functions

findChessboardCorners() #棋盘格角点检测
cornerSubPix() #亚像素检测
calibrateCamera() #单目标定 求解摄像机的内在参数和外在参数
stereoCalibrate() #双目标定 求解两个摄像头的内外参数矩阵，以及两个摄像头的位置关系R,T

code

import cv2
import os
import numpy as np

leftpath = 'images/left'
rightpath = 'images/right'
CHECKERBOARD = (11,12)  #棋盘格内角点数
square_size = (30,30)   #棋盘格大小，单位mm
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
imgpoints_l = []    #存放左图像坐标系下角点位置
imgpoints_r = []    #存放左图像坐标系下角点位置
objpoints = []   #存放世界坐标系下角点位置
objp = np.zeros((1, CHECKERBOARD[0]*CHECKERBOARD[1], 3), np.float32)
objp[0,:,:2] = np.mgrid[0:CHECKERBOARD[0], 0:CHECKERBOARD[1]].T.reshape(-1, 2)
objp[0,:,0] *= square_size[0]
objp[0,:,1] *= square_size[1]


for ii in os.listdir(leftpath):
    img_l = cv2.imread(os.path.join(leftpath,ii))
    gray_l = cv2.cvtColor(img_l,cv2.COLOR_BGR2GRAY)
    img_r = cv2.imread(os.path.join(rightpath,ii))
    gray_r = cv2.cvtColor(img_r,cv2.COLOR_BGR2GRAY)
    ret_l, corners_l = cv2.findChessboardCorners(gray_l, CHECKERBOARD)   #检测棋盘格内角点
    ret_r, corners_r = cv2.findChessboardCorners(gray_r, CHECKERBOARD)
    if ret_l and ret_r:
        objpoints.append(objp)
        corners2_l = cv2.cornerSubPix(gray_l,corners_l,(11,11),(-1,-1),criteria) 
        imgpoints_l.append(corners2_l)
        corners2_r = cv2.cornerSubPix(gray_r,corners_r,(11,11),(-1,-1),criteria)
        imgpoints_r.append(corners2_r)
        #img = cv2.drawChessboardCorners(img, CHECKERBOARD, corners2,ret)
        #cv2.imwrite('./ChessboardCornersimg.jpg', img)
ret, mtx_l, dist_l, rvecs_l, tvecs_l = cv2.calibrateCamera(objpoints, imgpoints_l, gray_l.shape[::-1],None,None)  #先分别做单目标定
ret, mtx_r, dist_r, rvecs_r, tvecs_r = cv2.calibrateCamera(objpoints, imgpoints_r, gray_r.shape[::-1],None,None)

retval, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, R, T, E, F = \
    cv2.stereoCalibrate(objpoints, imgpoints_l, imgpoints_r, mtx_l, dist_l, mtx_r, dist_r, gray_l.shape[::-1])   #再做双目标定

print("stereoCalibrate : \n")
print("Camera matrix left : \n")
print(cameraMatrix1)
print("distCoeffs left  : \n")
print(distCoeffs1)
print("cameraMatrix left : \n")
print(cameraMatrix2)
print("distCoeffs left : \n")
print(distCoeffs2)
print("R : \n")
print(R)
print("T : \n")
print(T)
print("E : \n")
print(E)
print("F : \n")
print(F)

Save the printed results to the calibration file

For epipolar correction
, refer to the blog machine vision learning notes (8) - Bouguet stereoscopic correction based on OpenCV and
the principle of Bouguet binocular stereoscopic correction from a white perspective.

Mainly used functions

stereoRectify() #计算旋转矩阵和投影矩阵
initUndistortRectifyMap() #计算校正查找映射表
remap() #重映射

code

import cv2
import numpy as np

def cat2images(limg, rimg):
    HEIGHT = limg.shape[0]
    WIDTH = limg.shape[1]
    imgcat = np.zeros((HEIGHT, WIDTH*2+20,3))
    imgcat[:,:WIDTH,:] = limg
    imgcat[:,-WIDTH:,:] = rimg
    for i in range(int(HEIGHT / 32)):
        imgcat[i*32,:,:] = 255 
    return imgcat

left_image = cv2.imread("images/left/268.jpg")
right_image = cv2.imread("images/right/268.jpg")

imgcat_source = cat2images(left_image,right_image)
HEIGHT = left_image.shape[0]
WIDTH = left_image.shape[1]
cv2.imwrite('imgcat_source.jpg', imgcat_source )

camera_matrix0 = np.array([[1.30991855e+03, 0.00000000e+00, 5.90463086e+02],
                            [0.00000000e+00, 1.31136722e+03, 3.33464608e+02],
                            [0.00000000e+00, 0.00000000e+00, 1.00000000e+00]]
                        ) .reshape((3,3)) #即上文标定得到的 cameraMatrix1
                        
distortion0 = np.array([-4.88890701e-01,  3.27964225e-01, -2.72130825e-04,  1.28030208e-03, -1.85964828e-01]) #即上文标定得到的 distCoeffs1

camera_matrix1 = np.array([[1.30057467e+03, 0.00000000e+00, 6.28445749e+02],
                            [0.00000000e+00, 1.30026325e+03, 3.90475091e+02],
                            [0.00000000e+00, 0.00000000e+00, 1.00000000e+00]]
                        ) .reshape((3,3)) #即上文标定得到的 cameraMatrix2
distortion1 = np.array([-4.95938411e-01,  2.70207629e-01,  1.81014753e-04, -4.58891345e-04, 4.41327829e-01]) #即上文标定得到的 distCoeffs2

R = np.array([[ 0.99989348,  0.01340678, -0.00576869], 
              [-0.01338004,  0.99989967,  0.00465071], 
              [ 0.00583046, -0.00457303,  0.99997255]]
            ) #即上文标定得到的 R
T = np.array([-244.28272039, 3.84124178, 2.0963191]) #即上文标定得到的T


(R_l, R_r, P_l, P_r, Q, validPixROI1, validPixROI2) = \
    cv2.stereoRectify(camera_matrix0, distortion0, camera_matrix1, distortion1, np.array([WIDTH,HEIGHT]), R, T) #计算旋转矩阵和投影矩阵

(map1, map2) = \
    cv2.initUndistortRectifyMap(camera_matrix0, distortion0, R_l, P_l, np.array([WIDTH,HEIGHT]), cv2.CV_32FC1) #计算校正查找映射表

rect_left_image = cv2.remap(left_image, map1, map2, cv2.INTER_CUBIC) #重映射

#左右图需要分别计算校正查找映射表以及重映射
(map1, map2) = \
    cv2.initUndistortRectifyMap(camera_matrix1, distortion1, R_r, P_r, np.array([WIDTH,HEIGHT]), cv2.CV_32FC1)
    
rect_right_image = cv2.remap(right_image, map1, map2, cv2.INTER_CUBIC)

imgcat_out = cat2images(rect_left_image,rect_right_image)
cv2.imwrite('imgcat_out.jpg', imgcat_out)

Effect picture
before correction
left picture

right picture

after correction