OpenCV-Python实践之ASIFT框架仿射特征匹配算法

Affine-SIFT仿射特征匹配算法简介

　　图像之间经过较大视差的匹配任务在于解决局部仿射不变特征的提取与描述，目前Harris/Hessian-Affine与MSER局部不变特征提取主要策略是采用拟合椭圆归一化方式。关于Harris-Affine与MSER-Affine算法都是首先提取尺度不变或区域中心(角、斑)点，然后归一化收敛到仿射变形6个参数估计，并不是完全的仿射估计方式，具体原理可以参考Harris-Affine仿射不变特征匹配算法与MSER仿射不变特征匹配算法。

　　Affine-SIFT算法不同于上述局部仿射算法，通过模拟相机镜头轴线与目标图像的所有成像夹角的方向参数：经度角与纬度角。通过经度角与纬度角来模拟图像所有视差下的仿射变形，然后使用SIFT算法来解决图像的尺度、平移旋转问题。简单概述一下Affine-SIFT算法的过程：

　　1 首先对待匹配两幅图像按照经度角、纬度角模式进行仿射变换插值重采样。

　　2 对变换后的图像进行SIFT算法特征匹配。

ASIFT算法模拟仿射变换匹配过程

　　其实，可以直观看到Affine-SIFT算法只是一个框架：模拟视差变换后的图像匹配解决尺度、平移、旋转的特征匹配算法可以更换掉SIFT算法。例如：Affine-SURF、Affine-BRISK、Affine-ORB等算法为了对Affine-SIFT算法进行加速的功能。以此来满足仿射视差下的实时性要求问题。

OpenCV-Python全仿射匹配代码

辅助功能common.py文件：计时函数

# Python 2/3 compatibility
from __future__ import print_function
import sys
PY3 = sys.version_info[0] == 3

if PY3:
    from functools import reduce

import numpy as np
import cv2 as cv

# built-in modules
import os
import itertools as it
from contextlib import contextmanager
def clock():
    return cv.getTickCount() / cv.getTickFrequency()

@contextmanager
def Timer(msg):
    print(msg, '...',)
    start = clock()
    try:
        yield
    finally:
        print("%.2f ms" % ((clock()-start)*1000))

辅助功能函数find_obj.py. 其中init_feature()为输入特征检测与描述子算法，分别有sift、surf、orb、akaze、brisk算法，然后进行匹配算法提纯特征点对！

# Python 2/3 compatibility
from __future__ import print_function

import numpy as np
import cv2 as cv
#from common import anorm, getsize

FLANN_INDEX_KDTREE = 1  # bug: flann enums are missing
FLANN_INDEX_LSH    = 6


def init_feature(name):
    chunks = name.split('-')
    if chunks[0] == 'sift':
        detector = cv.xfeatures2d.SIFT_create()
        norm = cv.NORM_L2
    elif chunks[0] == 'surf':
        detector = cv.xfeatures2d.SURF_create(800)
        norm = cv.NORM_L2
    elif chunks[0] == 'orb':
        detector = cv.ORB_create(400)
        norm = cv.NORM_HAMMING
    elif chunks[0] == 'akaze':
        detector = cv.AKAZE_create()
        norm = cv.NORM_HAMMING
    elif chunks[0] == 'brisk':
        detector = cv.BRISK_create()
        norm = cv.NORM_HAMMING
    else:
        return None, None
    if 'flann' in chunks:
        if norm == cv.NORM_L2:
            flann_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
        else:
            flann_params= dict(algorithm = FLANN_INDEX_LSH,
                               table_number = 6, # 12
                               key_size = 12,     # 20
                               multi_probe_level = 1) #2
        matcher = cv.FlannBasedMatcher(flann_params, {})  # bug : need to pass empty dict (#1329)
    else:
        matcher = cv.BFMatcher(norm)
    return detector, matcher


def filter_matches(kp1, kp2, matches, ratio = 0.75):
    mkp1, mkp2 = [], []
    for m in matches:
        if len(m) == 2 and m[0].distance < m[1].distance * ratio:
            m = m[0]
            mkp1.append(kp1[m.queryIdx])
            mkp2.append(kp2[m.trainIdx])
    p1 = np.float32([kp.pt for kp in mkp1])
    p2 = np.float32([kp.pt for kp in mkp2])
    kp_pairs = zip(mkp1, mkp2)
    return p1, p2, list(kp_pairs)

def explore_match(win, img1, img2, kp_pairs, status = None, H = None):
    h1, w1 = img1.shape[:2]
    h2, w2 = img2.shape[:2]
    vis = np.zeros((max(h1, h2), w1+w2), np.uint8)
    vis[:h1, :w1] = img1
    vis[:h2, w1:w1+w2] = img2
    vis = cv.cvtColor(vis, cv.COLOR_GRAY2BGR)

    if H is not None:
        corners = np.float32([[0, 0], [w1, 0], [w1, h1], [0, h1]])
        corners = np.int32( cv.perspectiveTransform(corners.reshape(1, -1, 2), H).reshape(-1, 2) + (w1, 0) )
        cv.polylines(vis, [corners], True, (255, 255, 255))

    if status is None:
        status = np.ones(len(kp_pairs), np.bool_)
    p1, p2 = [], []  # python 2 / python 3 change of zip unpacking
    for kpp in kp_pairs:
        p1.append(np.int32(kpp[0].pt))
        p2.append(np.int32(np.array(kpp[1].pt) + [w1, 0]))

    green = (0, 255, 0)
    red = (0, 0, 255)
    kp_color = (51, 103, 236)
    for (x1, y1), (x2, y2), inlier in zip(p1, p2, status):
        if inlier:
            col = green
            cv.circle(vis, (x1, y1), 2, col, -1)
            cv.circle(vis, (x2, y2), 2, col, -1)
        else:
            col = red
            r = 2
            thickness = 3
            cv.line(vis, (x1-r, y1-r), (x1+r, y1+r), col, thickness)
            cv.line(vis, (x1-r, y1+r), (x1+r, y1-r), col, thickness)
            cv.line(vis, (x2-r, y2-r), (x2+r, y2+r), col, thickness)
            cv.line(vis, (x2-r, y2+r), (x2+r, y2-r), col, thickness)
    vis0 = vis.copy()
    for (x1, y1), (x2, y2), inlier in zip(p1, p2, status):
        if inlier:
            cv.line(vis, (x1, y1), (x2, y2), green)

    cv.imshow(win, vis)

ASIFT算法框架主功能asift.py代码如下：主要参数设置
　　feature_name = opts.get(’–feature’, ‘brisk-flann’)中更换‘brisk-flann’中的brisk为sift、surf、orb、akaze等特征匹配算法,添加flann代表使用flann进行匹配，否则自动采用bf匹配算法。具体调用在功能辅助find_obj.py中init_feature()函数有具体细节。

# Python 2/3 compatibility
from __future__ import print_function

import numpy as np
import cv2 as cv

# built-in modules
import itertools as it
from multiprocessing.pool import ThreadPool

# local modules
from common import Timer
from find_obj import init_feature, filter_matches, explore_match


def affine_skew(tilt, phi, img, mask=None):
    '''
    affine_skew(tilt, phi, img, mask=None) -> skew_img, skew_mask, Ai

    Ai - is an affine transform matrix from skew_img to img
    '''
    h, w = img.shape[:2]
    if mask is None:
        mask = np.zeros((h, w), np.uint8)
        mask[:] = 255
    A = np.float32([[1, 0, 0], [0, 1, 0]])
    if phi != 0.0:
        phi = np.deg2rad(phi)
        s, c = np.sin(phi), np.cos(phi)
        A = np.float32([[c,-s], [ s, c]])
        corners = [[0, 0], [w, 0], [w, h], [0, h]]
        tcorners = np.int32( np.dot(corners, A.T) )
        x, y, w, h = cv.boundingRect(tcorners.reshape(1,-1,2))
        A = np.hstack([A, [[-x], [-y]]])
        img = cv.warpAffine(img, A, (w, h), flags=cv.INTER_LINEAR, borderMode=cv.BORDER_REPLICATE)
    if tilt != 1.0:
        s = 0.8*np.sqrt(tilt*tilt-1)
        img = cv.GaussianBlur(img, (0, 0), sigmaX=s, sigmaY=0.01)
        img = cv.resize(img, (0, 0), fx=1.0/tilt, fy=1.0, interpolation=cv.INTER_NEAREST)
        A[0] /= tilt
    if phi != 0.0 or tilt != 1.0:
        h, w = img.shape[:2]
        mask = cv.warpAffine(mask, A, (w, h), flags=cv.INTER_NEAREST)
    Ai = cv.invertAffineTransform(A)
    return img, mask, Ai


def affine_detect(detector, img, mask=None, pool=None):
    '''
    affine_detect(detector, img, mask=None, pool=None) -> keypoints, descrs

    Apply a set of affine transformations to the image, detect keypoints and
    reproject them into initial image coordinates.
    See http://www.ipol.im/pub/algo/my_affine_sift/ for the details.

    ThreadPool object may be passed to speedup the computation.
    '''
    params = [(1.0, 0.0)]
    for t in 2**(0.5*np.arange(1,6)):
        for phi in np.arange(0, 180, 72.0 / t):
            params.append((t, phi))

    def f(p):
        t, phi = p
        timg, tmask, Ai = affine_skew(t, phi, img)
        keypoints, descrs = detector.detectAndCompute(timg, tmask)
        for kp in keypoints:
            x, y = kp.pt
            kp.pt = tuple( np.dot(Ai, (x, y, 1)) )
        if descrs is None:
            descrs = []
        return keypoints, descrs

    keypoints, descrs = [], []
    if pool is None:
        ires = it.imap(f, params)
    else:
        ires = pool.imap(f, params)

    for i, (k, d) in enumerate(ires):
        print('affine sampling: %d / %d\r' % (i+1, len(params)), end='')
        keypoints.extend(k)
        descrs.extend(d)

    print()
    return keypoints, np.array(descrs)

if __name__ == '__main__':
    print(__doc__)

    import sys, getopt
    opts, args = getopt.getopt(sys.argv[1:], '', ['feature='])
    opts = dict(opts)
    '''
       --feature:  sift/surf/orb/akaze  
    '''
    feature_name = opts.get('--feature', 'brisk-flann')
    try:
        fn1, fn2 = args
    except:
        fn1 = './data/adam_zoom1_front.jpg'
        fn2 = './data/adam_zoom1_80deg.jpg'

    img1 = cv.imread(fn1, 0)
    img2 = cv.imread(fn2, 0)
    detector, matcher = init_feature(feature_name)

    if img1 is None:
        print('Failed to load fn1:', fn1)
        sys.exit(1)

    if img2 is None:
        print('Failed to load fn2:', fn2)
        sys.exit(1)

    if detector is None:
        print('unknown feature:', feature_name)
        sys.exit(1)
    print('using', feature_name)

    pool = ThreadPool(processes=cv.getNumberOfCPUs())
    kp1, desc1 = affine_detect(detector, img1, pool=pool)
    kp2, desc2 = affine_detect(detector, img2, pool=pool)
    print('img1 - %d features, img2 - %d features' % (len(kp1), len(kp2)))

    def match_and_draw(win):
        with Timer('matching'):
            raw_matches = matcher.knnMatch(desc1, trainDescriptors=desc2, k=2) #2
        p1, p2, kp_pairs = filter_matches(kp1, kp2, raw_matches)
        if len(p1) >= 4:
            H, status = cv.findHomography(p1, p2, cv.RANSAC, 5.0)
            print('%d / %d  inliers/matched' % (np.sum(status), len(status)))
            # do not draw outliers (there will be a lot of them)
            kp_pairs = [kpp for kpp, flag in zip(kp_pairs, status) if flag]
        else:
            H, status = None, None
            print('%d matches found, not enough for homography estimation' % len(p1))
        explore_match(win, img1, img2, kp_pairs, None, H)

    match_and_draw('affine find_obj')
    cv.waitKey()
    cv.destroyAllWindows()

实验对比结果

　　下图表中都是采用ASIFT算法框架进行全仿射图片采样，然后使用不同的特征匹配算法来完成仿射变换匹配，对特征点数、匹配时间、内联点对、匹配概率等。

Affine-SIFT特征匹配算法：

Affine-SURF特征匹配算法：

Affine-ORB特征匹配算法：

Affine-BRISK特征匹配算法：

Affine-AKAZE特征匹配算法：

小结

　　Affine-SIFT全仿射匹配算法框架，本文测试对比了全仿射框架算法：分别使用Affine-SURF、ORB、BRISK、AKAZE特征匹配算法。AKAZE算法耗时最短，性能较优！后续可以考虑将GMS运动网格估计算法替换RANSAC算法来加速精确特征匹配点对，这样可以进一步满足在SLAM三维的时间性能问题！每一个特征匹配环节进行逐步优化，来不断提升性能。如有错误，还请批评指正！

参考

http://www.cmap.polytechnique.fr/~yu/research/ASIFT/demo.html
https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_tutorials.html