オプティカルフローの一般的な可視化手法オプティカルフロー、オプティカルフロー画像生成

1。概要

オプティカルフローの一般的な可視化手法オプティカルフロー、オプティカルフロー画像生成

視覚化とワープ:

1. オプティカル フローは、画像間の対応するピクセルの変位を表し、
   通常、X 軸と Y 軸をそれぞれ表す 2 つのチャネルがあります。

一般的なオプティカル フロー サイズ: 高さ、幅、2

2. それを視覚化するにはどうすればよいでしょうか?

各ピクセルのオプティカルフローは、x と y の 2 つの方向を持ちます。角度は色相で表され、弾性率は彩度で表されます。
ここで他の人から学びましょう。同じ効果を持つ 2 つの方法があります。

2.方法 1 :

import numpy as np
# https://www.zhihu.com/question/500159852/answer/2233549991
# https://github.com/open-mmlab/mmcv/blob/c47c9196d067a0900b7b8987a8e82768edab2fff/mmcv/visualization/optflow.py#L76

def make_color_wheel(bins=None):
    """Build a color wheel.
    Args:
        bins(list or tuple, optional): Specify the number of bins for each
            color range, corresponding to six ranges: red -> yellow,
            yellow -> green, green -> cyan, cyan -> blue, blue -> magenta,
            magenta -> red. [15, 6, 4, 11, 13, 6] is used for default
            (see Middlebury).
    Returns:
        ndarray: Color wheel of shape (total_bins, 3).
    """
    if bins is None:
        bins = [15, 6, 4, 11, 13, 6]
    assert len(bins) == 6

    RY, YG, GC, CB, BM, MR = tuple(bins)
    print(RY)
    ry = [1, np.arange(RY) / RY, 0]
    yg = [1 - np.arange(YG) / YG, 1, 0]
    gc = [0, 1, np.arange(GC) / GC]
    cb = [0, 1 - np.arange(CB) / CB, 1]
    bm = [np.arange(BM) / BM, 0, 1]
    mr = [1, 0, 1 - np.arange(MR) / MR]
    print(ry)
    num_bins = RY + YG + GC + CB + BM + MR
    print(num_bins)
    color_wheel = np.zeros((3, num_bins), dtype=np.float32)
    print(color_wheel)

    col = 0
    for i, color in enumerate([ry, yg, gc, cb, bm, mr]):
        if i == 0:
            print(i, color)
        for j in range(3):
            color_wheel[j, col:col + bins[i]] = color[j]
        col += bins[i]

    return color_wheel.T

def flow2rgb(flow, color_wheel=None, unknown_thr=1e6):
    """Convert flow map to RGB image.

    Args:
        flow (ndarray): Array of optical flow.
        color_wheel (ndarray or None): Color wheel used to map flow field to
            RGB colorspace. Default color wheel will be used if not specified.
        unknown_thr (str): Values above this threshold will be marked as
            unknown and thus ignored.

    Returns:
        ndarray: RGB image that can be visualized.
    """
    assert flow.ndim == 3 and flow.shape[-1] == 2
    if color_wheel is None:
        color_wheel = make_color_wheel()
    assert color_wheel.ndim == 2 and color_wheel.shape[1] == 3
    num_bins = color_wheel.shape[0]
    dx = flow[:, :, 0].copy()
    dy = flow[:, :, 1].copy()

    ignore_inds = (
        np.isnan(dx) | np.isnan(dy) | (np.abs(dx) > unknown_thr) |
        (np.abs(dy) > unknown_thr))
    dx[ignore_inds] = 0
    dy[ignore_inds] = 0

    rad = np.sqrt(dx**2 + dy**2)  # HxW
    if np.any(rad > np.finfo(float).eps):
        max_rad = np.max(rad)  # 使用最大模长来放缩坐标值
        dx /= max_rad
        dy /= max_rad

    rad = np.sqrt(dx**2 + dy**2)  # HxW
    angle = np.arctan2(-dy, -dx) / np.pi  # HxW（-1, 1]

    bin_real = (angle + 1) / 2 * (num_bins - 1)  # HxW (0, num_bins-1]
    bin_left = np.floor(bin_real).astype(int)  # HxW 0,1,...,num_bins-1
    bin_right = (bin_left + 1) % num_bins  # HxW 1,2,...,num_bins % num_bins -> 1, 2, ..., num_bins, 0
    w = (bin_real - bin_left.astype(np.float32))[..., None]  # HxWx1
    flow_img = (1 - w) * color_wheel[bin_left, :] + w * color_wheel[bin_right, :]  # 线性插值计算实际的颜色值
    small_ind = rad <= 1  # 以模长为1作为分界线来分开处理，个人理解这里主要是用来控制颜色的饱和度，而前面的处理更像是控制色调。
    # 小于1的部分拉大
    flow_img[small_ind] = 1 - rad[small_ind, None] * (1 - flow_img[small_ind])
    # 大于1的部分缩小
    flow_img[np.logical_not(small_ind)] *= 0.75

    flow_img[ignore_inds, :] = 0

    return flow_img

3.方法 2 :

import numpy as np
from matplotlib import pyplot as plt


def make_color_wheel():
    """
    Generate color wheel according Middlebury color code
    :return: Color wheel
    """
    RY = 15
    YG = 6
    GC = 4
    CB = 11
    BM = 13
    MR = 6

    ncols = RY + YG + GC + CB + BM + MR

    colorwheel = np.zeros([ncols, 3])

    col = 0

    # RY
    colorwheel[0:RY, 0] = 255
    colorwheel[0:RY, 1] = np.transpose(np.floor(255*np.arange(0, RY) / RY))
    col += RY

    # YG
    colorwheel[col:col+YG, 0] = 255 - np.transpose(np.floor(255*np.arange(0, YG) / YG))
    colorwheel[col:col+YG, 1] = 255
    col += YG

    # GC
    colorwheel[col:col+GC, 1] = 255
    colorwheel[col:col+GC, 2] = np.transpose(np.floor(255*np.arange(0, GC) / GC))
    col += GC

    # CB
    colorwheel[col:col+CB, 1] = 255 - np.transpose(np.floor(255*np.arange(0, CB) / CB))
    colorwheel[col:col+CB, 2] = 255
    col += CB

    # BM
    colorwheel[col:col+BM, 2] = 255
    colorwheel[col:col+BM, 0] = np.transpose(np.floor(255*np.arange(0, BM) / BM))
    col += + BM

    # MR
    colorwheel[col:col+MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR))
    colorwheel[col:col+MR, 0] = 255

    return colorwheel

def compute_color(u, v):
    """
    compute optical flow color map
    :param u: optical flow horizontal map
    :param v: optical flow vertical map
    :return: optical flow in color code
    """
    [h, w] = u.shape
    img = np.zeros([h, w, 3])
    nanIdx = np.isnan(u) | np.isnan(v)
    u[nanIdx] = 0
    v[nanIdx] = 0

    colorwheel = make_color_wheel()
    ncols = np.size(colorwheel, 0)

    rad = np.sqrt(u**2+v**2)

    a = np.arctan2(-v, -u) / np.pi

    fk = (a+1) / 2 * (ncols - 1) + 1

    k0 = np.floor(fk).astype(int)

    k1 = k0 + 1
    k1[k1 == ncols+1] = 1
    f = fk - k0

    for i in range(0, np.size(colorwheel,1)):
        tmp = colorwheel[:, i]
        col0 = tmp[k0-1] / 255
        col1 = tmp[k1-1] / 255
        col = (1-f) * col0 + f * col1

        idx = rad <= 1
        col[idx] = 1-rad[idx]*(1-col[idx])
        notidx = np.logical_not(idx)

        col[notidx] *= 0.75
        img[:, :, i] = np.uint8(np.floor(255 * col*(1-nanIdx)))

    return img

def flow_to_image(flow):
    """
    Convert flow into middlebury color code image
    :param flow: optical flow map
    :return: optical flow image in middlebury color
    """
    u = flow[:, :, 0]
    v = flow[:, :, 1]

    maxu = -999.
    maxv = -999.
    minu = 999.
    minv = 999.
    UNKNOWN_FLOW_THRESH = 1e7
    SMALLFLOW = 0.0
    LARGEFLOW = 1e8

    idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH)
    u[idxUnknow] = 0
    v[idxUnknow] = 0

    maxu = max(maxu, np.max(u))
    minu = min(minu, np.min(u))

    maxv = max(maxv, np.max(v))
    minv = min(minv, np.min(v))

    rad = np.sqrt(u ** 2 + v ** 2)
    maxrad = max(-1, np.max(rad))

    u = u/(maxrad + np.finfo(float).eps)
    v = v/(maxrad + np.finfo(float).eps)

    img = compute_color(u, v)

    idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2)
    img[idx] = 0

    return np.uint8(img)

if __name__ == "__main__":
    print('flow')

4. 方法 3: トーチビジョンを使用する

from torchvision.utils import flow_to_image
import torch
def flow_to_image_torch(flow):
    flow = torch.from_numpy(np.transpose(flow, [2, 0, 1]))
    flow_im = flow_to_image(flow)
    img = np.transpose(flow_im.numpy(), [1, 2, 0])
    print(img.shape)
    return img

異なる色は方向を示し、深さは変位を示します。

5. 色や色合いは特定の意味を表現します

例を通じて分析するには、
中央から周囲に移動するオプティカル フロー マップを追加します。

X:[[-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]]
Y:[[-5. -5. -5. -5. -5. -5. -5. -5. -5. -5. -5.]
 [-4. -4. -4. -4. -4. -4. -4. -4. -4. -4. -4.]
 [-3. -3. -3. -3. -3. -3. -3. -3. -3. -3. -3.]
 [-2. -2. -2. -2. -2. -2. -2. -2. -2. -2. -2.]
 [-1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.]
 [-0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0.]
 [ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]
 [ 2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.]
 [ 3.  3.  3.  3.  3.  3.  3.  3.  3.  3.  3.]
 [ 4.  4.  4.  4.  4.  4.  4.  4.  4.  4.  4.]
 [ 5.  5.  5.  5.  5.  5.  5.  5.  5.  5.  5.]]

最初の 3 つの方法で生成されたオプティカル フロー グラフは一貫しています。

6. 方法 4: オフセット マップ

"""
箭头图
"""
def draw_flow(im, flow, step=40, norm=1):
    # 在间隔分开的像素采样点处绘制光流
    h, w = im.shape[:2]
    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2, -1).astype(int)
    if norm:
        fx, fy = flow[y, x].T / abs(flow[y, x]).max() * step // 2
    else:
        fx, fy = flow[y, x].T  # / flow[y, x].max() * step // 2
    # 创建线的终点
    ex = x + fx
    ey = y + fy
    lines = np.vstack([x, y, ex, ey]).T.reshape(-1, 2, 2)
    lines = lines.astype(np.uint32)
    # 创建图像并绘制
    vis = im.astype(np.uint8)  #cv2.cvtColor(im, cv2.COLOR_GRAY2BGR)
    for (x1, y1), (x2, y2) in lines:
        cv2.line(vis, (x1, y1), (x2, y2), (0, 255, 0), 2)
        cv2.circle(vis, (x1, y1), 2, (0, 0, 255), -1)
    return vis

7. 方法 5 のオプティカル フロー図（方法 1、2、3 と同様）

参考：デンスオプティカルフローの概要 オプティカルフローアルゴリズム（opencv Pythonコード付き）

def show_flow_hsv(flow, show_style=1):
    '''
    
    :param flow:
    :param show_style:
    :return:
    '''
    mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])#将直角坐标系光流场转成极坐标系

    hsv = np.zeros((flow.shape[0], flow.shape[1], 3), np.uint8)

    #光流可视化的颜色模式
    if show_style == 1:
        hsv[..., 0] = ang * 180 / np.pi / 2 #angle弧度转角度
        hsv[..., 1] = 255
        hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)#magnitude归到0～255之间
    elif show_style == 2:
        hsv[..., 0] = ang * 180 / np.pi / 2
        hsv[..., 1] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
        hsv[..., 2] = 255

    #hsv转bgr
    bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
    return bgr

8. テストコード

	flow_im1 = flow2rgb(flow, color_wheel=None, unknown_thr=1e6)
    flow_im2 = flow_to_image_torch(flow)
    flow_im3 = flow_to_image2(flow)
    flow_im4 = draw_flow(frame1, flow)
    flow_im5 = show_flow_hsv(flow, show_style=2)
    plt.figure()
    plt.subplot(231)
    plt.imshow(flow_im1)
    plt.subplot(232)
    plt.imshow(flow_im2)
    plt.subplot(233)
    plt.imshow(flow_im3)
    plt.subplot(234)
    plt.imshow(flow_im4[..., ::-1])
    plt.subplot(235)
    plt.imshow(flow_im5[..., ::-1])
    plt.show()

結果：

9. 一般的な使用方法3、4

def show_flow_im(frame, flow):
    h, w, c = frame.shape
    flow_im3 = flow_to_image_torch(flow)
    flow_im4 = draw_flow(frame, flow, min(h, w)/40)
    plt.figure()
    plt.subplot(121)
    plt.imshow(flow_im3)
    plt.subplot(122)
    plt.imshow(flow_im4)
    plt.show()

10. .floファイルの読み込み

def load_flow_to_numpy(path):
    with open(path, 'rb') as f:
        magic = np.fromfile(f, np.float32, count=1)
        assert (202021.25 == magic), 'Magic number incorrect. Invalid .flo file'
        w = np.fromfile(f, np.int32, count=1)[0]
        h = np.fromfile(f, np.int32, count=1)[0]
        data = np.fromfile(f, np.float32, count=2 * w * h)
    data2D = np.reshape(data, (h, w, 2))
    #print(data2D[:10,:10,0])
    return data2D

file_flow = r'D:\codeflow\MPI-Sintel-complete\training\flow\alley_1\frame_0001.flo'
flowd = load_flow_to_numpy(file_flow)
flow_gt = flow_to_image_torch(flowd)

11. メッシュ歪みマップ

import cv2
import numpy as np

import matplotlib.pyplot as plt
from PIL import Image

from matplotlib.collections import LineCollection

from opencv_flow import load_flow_to_numpy, show_flow_im


def plot_grid(x,y, ax=None, **kwargs):
    ax = ax or plt.gca()
    segs1 = np.stack((x,y), axis=2)
    segs2 = segs1.transpose(1,0,2)
    ax.add_collection(LineCollection(segs1, **kwargs))
    ax.add_collection(LineCollection(segs2, **kwargs))
    ax.autoscale()
    ax.invert_yaxis()  # y轴反向

def show_flow_grid( flow, img=None, step=30):
    '''
    :param flow:
    :param img:
    :param step:
    :return:
    '''
    h, w = flow.shape[:2]
    plt.figure()
    if img is not None:
        plt.imshow(img)

    grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h))
    plot_grid(grid_x[::step, ::step], grid_y[::step, ::step], color="lightgrey")
    grid_x2 = grid_x + flow[..., 0]
    grid_y2 = grid_y + flow[..., 1]

    plot_grid(grid_x2[::step, ::step], grid_y2[::step, ::step], color="C0")
    plt.show()
if __name__ == "__main__":
    import os

    os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
    file1 = r'D:\codeflow\MPI-Sintel-complete\training\final\alley_1\frame_0006.png'
    file2 = r'D:\codeflow\MPI-Sintel-complete\training\final\alley_1\frame_0007.png'
    file_flow = r'D:\codeflow\MPI-Sintel-complete\training\flow\alley_1\frame_0006.flo'

    # file1 = r'D:\codeflow\FlyingChairs2\train\0000016-img_0.png'
    # file2 = r'D:\codeflow\FlyingChairs2\train\0000016-img_1.png'
    # file_flow = r'D:\codeflow\FlyingChairs2\train\0000016-flow_01.flo'
    im1 = cv2.imread(file1)[..., ::-1]
    h, w, c = im1.shape
    im2 = cv2.resize(cv2.imread(file2), (w, h))[..., ::-1]
    flow_gt = load_flow_to_numpy(file_flow)
    flow_gt = cv2.resize(flow_gt, (w, h))

    show_flow_im(im1, flow_gt)

    show_flow_grid(flow_gt, im1)

画像：

12. アローグラフ

def quiver_flow(flow):
    """
    flow : np.float32, [h, w, 2], (x, y) format
    """
    h, w = flow.shape[:2]
    xx, yy = np.meshgrid(range(w), range(h))

    s = 40
    plt.figure()
    plt.quiver(xx[::s, ::s], yy[::s, ::s], flow[..., 0][::s, ::s], flow[..., 1][::s, ::s], color='green')
    plt.show()