Common visualization methods of optical flow optical flow, optical flow image generation

1 Overview

Common visualization methods of optical flow optical flow, optical flow image generation

Visualization and warp :

1. Optical flow represents the displacement of corresponding pixels between images.
   Generally, there are two channels, representing the X axis and the Y axis respectively.

Typical optical flow size: H, W, 2

2. How to visualize it?

The optical flow of each pixel has two directions of x and y. The angle is represented by hue, and the modulus is represented by saturation.
Learn from others here. There are two methods with the same effect.

2. Method 1 :

import numpy as np
# https://www.zhihu.com/question/500159852/answer/2233549991
# https://github.com/open-mmlab/mmcv/blob/c47c9196d067a0900b7b8987a8e82768edab2fff/mmcv/visualization/optflow.py#L76

def make_color_wheel(bins=None):
    """Build a color wheel.
    Args:
        bins(list or tuple, optional): Specify the number of bins for each
            color range, corresponding to six ranges: red -> yellow,
            yellow -> green, green -> cyan, cyan -> blue, blue -> magenta,
            magenta -> red. [15, 6, 4, 11, 13, 6] is used for default
            (see Middlebury).
    Returns:
        ndarray: Color wheel of shape (total_bins, 3).
    """
    if bins is None:
        bins = [15, 6, 4, 11, 13, 6]
    assert len(bins) == 6

    RY, YG, GC, CB, BM, MR = tuple(bins)
    print(RY)
    ry = [1, np.arange(RY) / RY, 0]
    yg = [1 - np.arange(YG) / YG, 1, 0]
    gc = [0, 1, np.arange(GC) / GC]
    cb = [0, 1 - np.arange(CB) / CB, 1]
    bm = [np.arange(BM) / BM, 0, 1]
    mr = [1, 0, 1 - np.arange(MR) / MR]
    print(ry)
    num_bins = RY + YG + GC + CB + BM + MR
    print(num_bins)
    color_wheel = np.zeros((3, num_bins), dtype=np.float32)
    print(color_wheel)

    col = 0
    for i, color in enumerate([ry, yg, gc, cb, bm, mr]):
        if i == 0:
            print(i, color)
        for j in range(3):
            color_wheel[j, col:col + bins[i]] = color[j]
        col += bins[i]

    return color_wheel.T

def flow2rgb(flow, color_wheel=None, unknown_thr=1e6):
    """Convert flow map to RGB image.

    Args:
        flow (ndarray): Array of optical flow.
        color_wheel (ndarray or None): Color wheel used to map flow field to
            RGB colorspace. Default color wheel will be used if not specified.
        unknown_thr (str): Values above this threshold will be marked as
            unknown and thus ignored.

    Returns:
        ndarray: RGB image that can be visualized.
    """
    assert flow.ndim == 3 and flow.shape[-1] == 2
    if color_wheel is None:
        color_wheel = make_color_wheel()
    assert color_wheel.ndim == 2 and color_wheel.shape[1] == 3
    num_bins = color_wheel.shape[0]
    dx = flow[:, :, 0].copy()
    dy = flow[:, :, 1].copy()

    ignore_inds = (
        np.isnan(dx) | np.isnan(dy) | (np.abs(dx) > unknown_thr) |
        (np.abs(dy) > unknown_thr))
    dx[ignore_inds] = 0
    dy[ignore_inds] = 0

    rad = np.sqrt(dx**2 + dy**2)  # HxW
    if np.any(rad > np.finfo(float).eps):
        max_rad = np.max(rad)  # 使用最大模长来放缩坐标值
        dx /= max_rad
        dy /= max_rad

    rad = np.sqrt(dx**2 + dy**2)  # HxW
    angle = np.arctan2(-dy, -dx) / np.pi  # HxW（-1, 1]

    bin_real = (angle + 1) / 2 * (num_bins - 1)  # HxW (0, num_bins-1]
    bin_left = np.floor(bin_real).astype(int)  # HxW 0,1,...,num_bins-1
    bin_right = (bin_left + 1) % num_bins  # HxW 1,2,...,num_bins % num_bins -> 1, 2, ..., num_bins, 0
    w = (bin_real - bin_left.astype(np.float32))[..., None]  # HxWx1
    flow_img = (1 - w) * color_wheel[bin_left, :] + w * color_wheel[bin_right, :]  # 线性插值计算实际的颜色值
    small_ind = rad <= 1  # 以模长为1作为分界线来分开处理，个人理解这里主要是用来控制颜色的饱和度，而前面的处理更像是控制色调。
    # 小于1的部分拉大
    flow_img[small_ind] = 1 - rad[small_ind, None] * (1 - flow_img[small_ind])
    # 大于1的部分缩小
    flow_img[np.logical_not(small_ind)] *= 0.75

    flow_img[ignore_inds, :] = 0

    return flow_img

3. Method 2 :

import numpy as np
from matplotlib import pyplot as plt


def make_color_wheel():
    """
    Generate color wheel according Middlebury color code
    :return: Color wheel
    """
    RY = 15
    YG = 6
    GC = 4
    CB = 11
    BM = 13
    MR = 6

    ncols = RY + YG + GC + CB + BM + MR

    colorwheel = np.zeros([ncols, 3])

    col = 0

    # RY
    colorwheel[0:RY, 0] = 255
    colorwheel[0:RY, 1] = np.transpose(np.floor(255*np.arange(0, RY) / RY))
    col += RY

    # YG
    colorwheel[col:col+YG, 0] = 255 - np.transpose(np.floor(255*np.arange(0, YG) / YG))
    colorwheel[col:col+YG, 1] = 255
    col += YG

    # GC
    colorwheel[col:col+GC, 1] = 255
    colorwheel[col:col+GC, 2] = np.transpose(np.floor(255*np.arange(0, GC) / GC))
    col += GC

    # CB
    colorwheel[col:col+CB, 1] = 255 - np.transpose(np.floor(255*np.arange(0, CB) / CB))
    colorwheel[col:col+CB, 2] = 255
    col += CB

    # BM
    colorwheel[col:col+BM, 2] = 255
    colorwheel[col:col+BM, 0] = np.transpose(np.floor(255*np.arange(0, BM) / BM))
    col += + BM

    # MR
    colorwheel[col:col+MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR))
    colorwheel[col:col+MR, 0] = 255

    return colorwheel

def compute_color(u, v):
    """
    compute optical flow color map
    :param u: optical flow horizontal map
    :param v: optical flow vertical map
    :return: optical flow in color code
    """
    [h, w] = u.shape
    img = np.zeros([h, w, 3])
    nanIdx = np.isnan(u) | np.isnan(v)
    u[nanIdx] = 0
    v[nanIdx] = 0

    colorwheel = make_color_wheel()
    ncols = np.size(colorwheel, 0)

    rad = np.sqrt(u**2+v**2)

    a = np.arctan2(-v, -u) / np.pi

    fk = (a+1) / 2 * (ncols - 1) + 1

    k0 = np.floor(fk).astype(int)

    k1 = k0 + 1
    k1[k1 == ncols+1] = 1
    f = fk - k0

    for i in range(0, np.size(colorwheel,1)):
        tmp = colorwheel[:, i]
        col0 = tmp[k0-1] / 255
        col1 = tmp[k1-1] / 255
        col = (1-f) * col0 + f * col1

        idx = rad <= 1
        col[idx] = 1-rad[idx]*(1-col[idx])
        notidx = np.logical_not(idx)

        col[notidx] *= 0.75
        img[:, :, i] = np.uint8(np.floor(255 * col*(1-nanIdx)))

    return img

def flow_to_image(flow):
    """
    Convert flow into middlebury color code image
    :param flow: optical flow map
    :return: optical flow image in middlebury color
    """
    u = flow[:, :, 0]
    v = flow[:, :, 1]

    maxu = -999.
    maxv = -999.
    minu = 999.
    minv = 999.
    UNKNOWN_FLOW_THRESH = 1e7
    SMALLFLOW = 0.0
    LARGEFLOW = 1e8

    idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH)
    u[idxUnknow] = 0
    v[idxUnknow] = 0

    maxu = max(maxu, np.max(u))
    minu = min(minu, np.min(u))

    maxv = max(maxv, np.max(v))
    minv = min(minv, np.min(v))

    rad = np.sqrt(u ** 2 + v ** 2)
    maxrad = max(-1, np.max(rad))

    u = u/(maxrad + np.finfo(float).eps)
    v = v/(maxrad + np.finfo(float).eps)

    img = compute_color(u, v)

    idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2)
    img[idx] = 0

    return np.uint8(img)

if __name__ == "__main__":
    print('flow')

4. Method 3: Using torchvision

from torchvision.utils import flow_to_image
import torch
def flow_to_image_torch(flow):
    flow = torch.from_numpy(np.transpose(flow, [2, 0, 1]))
    flow_im = flow_to_image(flow)
    img = np.transpose(flow_im.numpy(), [1, 2, 0])
    print(img.shape)
    return img

Different colors indicate the direction, and the depth indicates the displacement

5. Colors and shades express specific meanings

To analyze through an example,
add an optical flow map that is shifted from the middle to the surrounding:

X:[[-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]]
Y:[[-5. -5. -5. -5. -5. -5. -5. -5. -5. -5. -5.]
 [-4. -4. -4. -4. -4. -4. -4. -4. -4. -4. -4.]
 [-3. -3. -3. -3. -3. -3. -3. -3. -3. -3. -3.]
 [-2. -2. -2. -2. -2. -2. -2. -2. -2. -2. -2.]
 [-1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.]
 [-0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0.]
 [ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]
 [ 2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.]
 [ 3.  3.  3.  3.  3.  3.  3.  3.  3.  3.  3.]
 [ 4.  4.  4.  4.  4.  4.  4.  4.  4.  4.  4.]
 [ 5.  5.  5.  5.  5.  5.  5.  5.  5.  5.  5.]]

The optical flow graphs generated by the first three methods are consistent:

6. Method 4: Offset Map

"""
箭头图
"""
def draw_flow(im, flow, step=40, norm=1):
    # 在间隔分开的像素采样点处绘制光流
    h, w = im.shape[:2]
    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2, -1).astype(int)
    if norm:
        fx, fy = flow[y, x].T / abs(flow[y, x]).max() * step // 2
    else:
        fx, fy = flow[y, x].T  # / flow[y, x].max() * step // 2
    # 创建线的终点
    ex = x + fx
    ey = y + fy
    lines = np.vstack([x, y, ex, ey]).T.reshape(-1, 2, 2)
    lines = lines.astype(np.uint32)
    # 创建图像并绘制
    vis = im.astype(np.uint8)  #cv2.cvtColor(im, cv2.COLOR_GRAY2BGR)
    for (x1, y1), (x2, y2) in lines:
        cv2.line(vis, (x1, y1), (x2, y2), (0, 255, 0), 2)
        cv2.circle(vis, (x1, y1), 2, (0, 0, 255), -1)
    return vis

7. Method five optical flow diagram, similar to method one two three

Reference: Summary of dense optical flow Optical Flow algorithm (with opencv python code)

def show_flow_hsv(flow, show_style=1):
    '''
    
    :param flow:
    :param show_style:
    :return:
    '''
    mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])#将直角坐标系光流场转成极坐标系

    hsv = np.zeros((flow.shape[0], flow.shape[1], 3), np.uint8)

    #光流可视化的颜色模式
    if show_style == 1:
        hsv[..., 0] = ang * 180 / np.pi / 2 #angle弧度转角度
        hsv[..., 1] = 255
        hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)#magnitude归到0～255之间
    elif show_style == 2:
        hsv[..., 0] = ang * 180 / np.pi / 2
        hsv[..., 1] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
        hsv[..., 2] = 255

    #hsv转bgr
    bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
    return bgr

8. Test code

	flow_im1 = flow2rgb(flow, color_wheel=None, unknown_thr=1e6)
    flow_im2 = flow_to_image_torch(flow)
    flow_im3 = flow_to_image2(flow)
    flow_im4 = draw_flow(frame1, flow)
    flow_im5 = show_flow_hsv(flow, show_style=2)
    plt.figure()
    plt.subplot(231)
    plt.imshow(flow_im1)
    plt.subplot(232)
    plt.imshow(flow_im2)
    plt.subplot(233)
    plt.imshow(flow_im3)
    plt.subplot(234)
    plt.imshow(flow_im4[..., ::-1])
    plt.subplot(235)
    plt.imshow(flow_im5[..., ::-1])
    plt.show()

result:

9. General usage methods 3 and 4

def show_flow_im(frame, flow):
    h, w, c = frame.shape
    flow_im3 = flow_to_image_torch(flow)
    flow_im4 = draw_flow(frame, flow, min(h, w)/40)
    plt.figure()
    plt.subplot(121)
    plt.imshow(flow_im3)
    plt.subplot(122)
    plt.imshow(flow_im4)
    plt.show()

10. .flo file reading

def load_flow_to_numpy(path):
    with open(path, 'rb') as f:
        magic = np.fromfile(f, np.float32, count=1)
        assert (202021.25 == magic), 'Magic number incorrect. Invalid .flo file'
        w = np.fromfile(f, np.int32, count=1)[0]
        h = np.fromfile(f, np.int32, count=1)[0]
        data = np.fromfile(f, np.float32, count=2 * w * h)
    data2D = np.reshape(data, (h, w, 2))
    #print(data2D[:10,:10,0])
    return data2D

file_flow = r'D:\codeflow\MPI-Sintel-complete\training\flow\alley_1\frame_0001.flo'
flowd = load_flow_to_numpy(file_flow)
flow_gt = flow_to_image_torch(flowd)

11. Mesh Distortion Map

import cv2
import numpy as np

import matplotlib.pyplot as plt
from PIL import Image

from matplotlib.collections import LineCollection

from opencv_flow import load_flow_to_numpy, show_flow_im


def plot_grid(x,y, ax=None, **kwargs):
    ax = ax or plt.gca()
    segs1 = np.stack((x,y), axis=2)
    segs2 = segs1.transpose(1,0,2)
    ax.add_collection(LineCollection(segs1, **kwargs))
    ax.add_collection(LineCollection(segs2, **kwargs))
    ax.autoscale()
    ax.invert_yaxis()  # y轴反向

def show_flow_grid( flow, img=None, step=30):
    '''
    :param flow:
    :param img:
    :param step:
    :return:
    '''
    h, w = flow.shape[:2]
    plt.figure()
    if img is not None:
        plt.imshow(img)

    grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h))
    plot_grid(grid_x[::step, ::step], grid_y[::step, ::step], color="lightgrey")
    grid_x2 = grid_x + flow[..., 0]
    grid_y2 = grid_y + flow[..., 1]

    plot_grid(grid_x2[::step, ::step], grid_y2[::step, ::step], color="C0")
    plt.show()
if __name__ == "__main__":
    import os

    os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
    file1 = r'D:\codeflow\MPI-Sintel-complete\training\final\alley_1\frame_0006.png'
    file2 = r'D:\codeflow\MPI-Sintel-complete\training\final\alley_1\frame_0007.png'
    file_flow = r'D:\codeflow\MPI-Sintel-complete\training\flow\alley_1\frame_0006.flo'

    # file1 = r'D:\codeflow\FlyingChairs2\train\0000016-img_0.png'
    # file2 = r'D:\codeflow\FlyingChairs2\train\0000016-img_1.png'
    # file_flow = r'D:\codeflow\FlyingChairs2\train\0000016-flow_01.flo'
    im1 = cv2.imread(file1)[..., ::-1]
    h, w, c = im1.shape
    im2 = cv2.resize(cv2.imread(file2), (w, h))[..., ::-1]
    flow_gt = load_flow_to_numpy(file_flow)
    flow_gt = cv2.resize(flow_gt, (w, h))

    show_flow_im(im1, flow_gt)

    show_flow_grid(flow_gt, im1)

image:

12. Arrow graph

def quiver_flow(flow):
    """
    flow : np.float32, [h, w, 2], (x, y) format
    """
    h, w = flow.shape[:2]
    xx, yy = np.meshgrid(range(w), range(h))

    s = 40
    plt.figure()
    plt.quiver(xx[::s, ::s], yy[::s, ::s], flow[..., 0][::s, ::s], flow[..., 1][::s, ::s], color='green')
    plt.show()