Métodos comunes de visualización de flujo óptico flujo óptico, generación de imágenes de flujo óptico

1. Información general

Métodos comunes de visualización de flujo óptico flujo óptico, generación de imágenes de flujo óptico

Visualización y deformación :

1. El flujo óptico representa el desplazamiento de los píxeles correspondientes entre las imágenes.
   Generalmente, hay dos canales, que representan el eje X y el eje Y respectivamente.

Tamaño de flujo óptico típico: H, W, 2

2. ¿Cómo visualizarlo?

El flujo óptico de cada píxel tiene dos direcciones de x e y. El ángulo está representado por el tono y el módulo está representado por la saturación.
Aprende de otros aquí. Hay dos métodos con el mismo efecto.

2. Método 1 :

import numpy as np
# https://www.zhihu.com/question/500159852/answer/2233549991
# https://github.com/open-mmlab/mmcv/blob/c47c9196d067a0900b7b8987a8e82768edab2fff/mmcv/visualization/optflow.py#L76

def make_color_wheel(bins=None):
    """Build a color wheel.
    Args:
        bins(list or tuple, optional): Specify the number of bins for each
            color range, corresponding to six ranges: red -> yellow,
            yellow -> green, green -> cyan, cyan -> blue, blue -> magenta,
            magenta -> red. [15, 6, 4, 11, 13, 6] is used for default
            (see Middlebury).
    Returns:
        ndarray: Color wheel of shape (total_bins, 3).
    """
    if bins is None:
        bins = [15, 6, 4, 11, 13, 6]
    assert len(bins) == 6

    RY, YG, GC, CB, BM, MR = tuple(bins)
    print(RY)
    ry = [1, np.arange(RY) / RY, 0]
    yg = [1 - np.arange(YG) / YG, 1, 0]
    gc = [0, 1, np.arange(GC) / GC]
    cb = [0, 1 - np.arange(CB) / CB, 1]
    bm = [np.arange(BM) / BM, 0, 1]
    mr = [1, 0, 1 - np.arange(MR) / MR]
    print(ry)
    num_bins = RY + YG + GC + CB + BM + MR
    print(num_bins)
    color_wheel = np.zeros((3, num_bins), dtype=np.float32)
    print(color_wheel)

    col = 0
    for i, color in enumerate([ry, yg, gc, cb, bm, mr]):
        if i == 0:
            print(i, color)
        for j in range(3):
            color_wheel[j, col:col + bins[i]] = color[j]
        col += bins[i]

    return color_wheel.T

def flow2rgb(flow, color_wheel=None, unknown_thr=1e6):
    """Convert flow map to RGB image.

    Args:
        flow (ndarray): Array of optical flow.
        color_wheel (ndarray or None): Color wheel used to map flow field to
            RGB colorspace. Default color wheel will be used if not specified.
        unknown_thr (str): Values above this threshold will be marked as
            unknown and thus ignored.

    Returns:
        ndarray: RGB image that can be visualized.
    """
    assert flow.ndim == 3 and flow.shape[-1] == 2
    if color_wheel is None:
        color_wheel = make_color_wheel()
    assert color_wheel.ndim == 2 and color_wheel.shape[1] == 3
    num_bins = color_wheel.shape[0]
    dx = flow[:, :, 0].copy()
    dy = flow[:, :, 1].copy()

    ignore_inds = (
        np.isnan(dx) | np.isnan(dy) | (np.abs(dx) > unknown_thr) |
        (np.abs(dy) > unknown_thr))
    dx[ignore_inds] = 0
    dy[ignore_inds] = 0

    rad = np.sqrt(dx**2 + dy**2)  # HxW
    if np.any(rad > np.finfo(float).eps):
        max_rad = np.max(rad)  # 使用最大模长来放缩坐标值
        dx /= max_rad
        dy /= max_rad

    rad = np.sqrt(dx**2 + dy**2)  # HxW
    angle = np.arctan2(-dy, -dx) / np.pi  # HxW（-1, 1]

    bin_real = (angle + 1) / 2 * (num_bins - 1)  # HxW (0, num_bins-1]
    bin_left = np.floor(bin_real).astype(int)  # HxW 0,1,...,num_bins-1
    bin_right = (bin_left + 1) % num_bins  # HxW 1,2,...,num_bins % num_bins -> 1, 2, ..., num_bins, 0
    w = (bin_real - bin_left.astype(np.float32))[..., None]  # HxWx1
    flow_img = (1 - w) * color_wheel[bin_left, :] + w * color_wheel[bin_right, :]  # 线性插值计算实际的颜色值
    small_ind = rad <= 1  # 以模长为1作为分界线来分开处理，个人理解这里主要是用来控制颜色的饱和度，而前面的处理更像是控制色调。
    # 小于1的部分拉大
    flow_img[small_ind] = 1 - rad[small_ind, None] * (1 - flow_img[small_ind])
    # 大于1的部分缩小
    flow_img[np.logical_not(small_ind)] *= 0.75

    flow_img[ignore_inds, :] = 0

    return flow_img

3. Método 2 :

import numpy as np
from matplotlib import pyplot as plt


def make_color_wheel():
    """
    Generate color wheel according Middlebury color code
    :return: Color wheel
    """
    RY = 15
    YG = 6
    GC = 4
    CB = 11
    BM = 13
    MR = 6

    ncols = RY + YG + GC + CB + BM + MR

    colorwheel = np.zeros([ncols, 3])

    col = 0

    # RY
    colorwheel[0:RY, 0] = 255
    colorwheel[0:RY, 1] = np.transpose(np.floor(255*np.arange(0, RY) / RY))
    col += RY

    # YG
    colorwheel[col:col+YG, 0] = 255 - np.transpose(np.floor(255*np.arange(0, YG) / YG))
    colorwheel[col:col+YG, 1] = 255
    col += YG

    # GC
    colorwheel[col:col+GC, 1] = 255
    colorwheel[col:col+GC, 2] = np.transpose(np.floor(255*np.arange(0, GC) / GC))
    col += GC

    # CB
    colorwheel[col:col+CB, 1] = 255 - np.transpose(np.floor(255*np.arange(0, CB) / CB))
    colorwheel[col:col+CB, 2] = 255
    col += CB

    # BM
    colorwheel[col:col+BM, 2] = 255
    colorwheel[col:col+BM, 0] = np.transpose(np.floor(255*np.arange(0, BM) / BM))
    col += + BM

    # MR
    colorwheel[col:col+MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR))
    colorwheel[col:col+MR, 0] = 255

    return colorwheel

def compute_color(u, v):
    """
    compute optical flow color map
    :param u: optical flow horizontal map
    :param v: optical flow vertical map
    :return: optical flow in color code
    """
    [h, w] = u.shape
    img = np.zeros([h, w, 3])
    nanIdx = np.isnan(u) | np.isnan(v)
    u[nanIdx] = 0
    v[nanIdx] = 0

    colorwheel = make_color_wheel()
    ncols = np.size(colorwheel, 0)

    rad = np.sqrt(u**2+v**2)

    a = np.arctan2(-v, -u) / np.pi

    fk = (a+1) / 2 * (ncols - 1) + 1

    k0 = np.floor(fk).astype(int)

    k1 = k0 + 1
    k1[k1 == ncols+1] = 1
    f = fk - k0

    for i in range(0, np.size(colorwheel,1)):
        tmp = colorwheel[:, i]
        col0 = tmp[k0-1] / 255
        col1 = tmp[k1-1] / 255
        col = (1-f) * col0 + f * col1

        idx = rad <= 1
        col[idx] = 1-rad[idx]*(1-col[idx])
        notidx = np.logical_not(idx)

        col[notidx] *= 0.75
        img[:, :, i] = np.uint8(np.floor(255 * col*(1-nanIdx)))

    return img

def flow_to_image(flow):
    """
    Convert flow into middlebury color code image
    :param flow: optical flow map
    :return: optical flow image in middlebury color
    """
    u = flow[:, :, 0]
    v = flow[:, :, 1]

    maxu = -999.
    maxv = -999.
    minu = 999.
    minv = 999.
    UNKNOWN_FLOW_THRESH = 1e7
    SMALLFLOW = 0.0
    LARGEFLOW = 1e8

    idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH)
    u[idxUnknow] = 0
    v[idxUnknow] = 0

    maxu = max(maxu, np.max(u))
    minu = min(minu, np.min(u))

    maxv = max(maxv, np.max(v))
    minv = min(minv, np.min(v))

    rad = np.sqrt(u ** 2 + v ** 2)
    maxrad = max(-1, np.max(rad))

    u = u/(maxrad + np.finfo(float).eps)
    v = v/(maxrad + np.finfo(float).eps)

    img = compute_color(u, v)

    idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2)
    img[idx] = 0

    return np.uint8(img)

if __name__ == "__main__":
    print('flow')

4. Método 3: Uso de torchvision

from torchvision.utils import flow_to_image
import torch
def flow_to_image_torch(flow):
    flow = torch.from_numpy(np.transpose(flow, [2, 0, 1]))
    flow_im = flow_to_image(flow)
    img = np.transpose(flow_im.numpy(), [1, 2, 0])
    print(img.shape)
    return img

Los diferentes colores indican la dirección y la profundidad indica el desplazamiento.

5. Los colores y las sombras expresan significados específicos

Para analizar a través de un ejemplo,
agregue un mapa de flujo óptico que se desplaza desde el medio hacia los alrededores:

X:[[-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]
 [-5. -4. -3. -2. -1. -0.  1.  2.  3.  4.  5.]]
Y:[[-5. -5. -5. -5. -5. -5. -5. -5. -5. -5. -5.]
 [-4. -4. -4. -4. -4. -4. -4. -4. -4. -4. -4.]
 [-3. -3. -3. -3. -3. -3. -3. -3. -3. -3. -3.]
 [-2. -2. -2. -2. -2. -2. -2. -2. -2. -2. -2.]
 [-1. -1. -1. -1. -1. -1. -1. -1. -1. -1. -1.]
 [-0. -0. -0. -0. -0. -0. -0. -0. -0. -0. -0.]
 [ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]
 [ 2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.]
 [ 3.  3.  3.  3.  3.  3.  3.  3.  3.  3.  3.]
 [ 4.  4.  4.  4.  4.  4.  4.  4.  4.  4.  4.]
 [ 5.  5.  5.  5.  5.  5.  5.  5.  5.  5.  5.]]

Los gráficos de flujo óptico generados por los primeros tres métodos son consistentes:

6. Método 4: Mapa de compensación

"""
箭头图
"""
def draw_flow(im, flow, step=40, norm=1):
    # 在间隔分开的像素采样点处绘制光流
    h, w = im.shape[:2]
    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2, -1).astype(int)
    if norm:
        fx, fy = flow[y, x].T / abs(flow[y, x]).max() * step // 2
    else:
        fx, fy = flow[y, x].T  # / flow[y, x].max() * step // 2
    # 创建线的终点
    ex = x + fx
    ey = y + fy
    lines = np.vstack([x, y, ex, ey]).T.reshape(-1, 2, 2)
    lines = lines.astype(np.uint32)
    # 创建图像并绘制
    vis = im.astype(np.uint8)  #cv2.cvtColor(im, cv2.COLOR_GRAY2BGR)
    for (x1, y1), (x2, y2) in lines:
        cv2.line(vis, (x1, y1), (x2, y2), (0, 255, 0), 2)
        cv2.circle(vis, (x1, y1), 2, (0, 0, 255), -1)
    return vis

7. Diagrama de flujo óptico del método cinco, similar al método uno dos tres

Referencia: Resumen del algoritmo de flujo óptico denso de flujo óptico (con código python opencv)

def show_flow_hsv(flow, show_style=1):
    '''
    
    :param flow:
    :param show_style:
    :return:
    '''
    mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])#将直角坐标系光流场转成极坐标系

    hsv = np.zeros((flow.shape[0], flow.shape[1], 3), np.uint8)

    #光流可视化的颜色模式
    if show_style == 1:
        hsv[..., 0] = ang * 180 / np.pi / 2 #angle弧度转角度
        hsv[..., 1] = 255
        hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)#magnitude归到0～255之间
    elif show_style == 2:
        hsv[..., 0] = ang * 180 / np.pi / 2
        hsv[..., 1] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
        hsv[..., 2] = 255

    #hsv转bgr
    bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
    return bgr

8. Código de prueba

	flow_im1 = flow2rgb(flow, color_wheel=None, unknown_thr=1e6)
    flow_im2 = flow_to_image_torch(flow)
    flow_im3 = flow_to_image2(flow)
    flow_im4 = draw_flow(frame1, flow)
    flow_im5 = show_flow_hsv(flow, show_style=2)
    plt.figure()
    plt.subplot(231)
    plt.imshow(flow_im1)
    plt.subplot(232)
    plt.imshow(flow_im2)
    plt.subplot(233)
    plt.imshow(flow_im3)
    plt.subplot(234)
    plt.imshow(flow_im4[..., ::-1])
    plt.subplot(235)
    plt.imshow(flow_im5[..., ::-1])
    plt.show()

resultado:

9. Métodos de uso general 3 y 4

def show_flow_im(frame, flow):
    h, w, c = frame.shape
    flow_im3 = flow_to_image_torch(flow)
    flow_im4 = draw_flow(frame, flow, min(h, w)/40)
    plt.figure()
    plt.subplot(121)
    plt.imshow(flow_im3)
    plt.subplot(122)
    plt.imshow(flow_im4)
    plt.show()

10. lectura de archivos .flo

def load_flow_to_numpy(path):
    with open(path, 'rb') as f:
        magic = np.fromfile(f, np.float32, count=1)
        assert (202021.25 == magic), 'Magic number incorrect. Invalid .flo file'
        w = np.fromfile(f, np.int32, count=1)[0]
        h = np.fromfile(f, np.int32, count=1)[0]
        data = np.fromfile(f, np.float32, count=2 * w * h)
    data2D = np.reshape(data, (h, w, 2))
    #print(data2D[:10,:10,0])
    return data2D

file_flow = r'D:\codeflow\MPI-Sintel-complete\training\flow\alley_1\frame_0001.flo'
flowd = load_flow_to_numpy(file_flow)
flow_gt = flow_to_image_torch(flowd)

11. Mapa de distorsión de malla

import cv2
import numpy as np

import matplotlib.pyplot as plt
from PIL import Image

from matplotlib.collections import LineCollection

from opencv_flow import load_flow_to_numpy, show_flow_im


def plot_grid(x,y, ax=None, **kwargs):
    ax = ax or plt.gca()
    segs1 = np.stack((x,y), axis=2)
    segs2 = segs1.transpose(1,0,2)
    ax.add_collection(LineCollection(segs1, **kwargs))
    ax.add_collection(LineCollection(segs2, **kwargs))
    ax.autoscale()
    ax.invert_yaxis()  # y轴反向

def show_flow_grid( flow, img=None, step=30):
    '''
    :param flow:
    :param img:
    :param step:
    :return:
    '''
    h, w = flow.shape[:2]
    plt.figure()
    if img is not None:
        plt.imshow(img)

    grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h))
    plot_grid(grid_x[::step, ::step], grid_y[::step, ::step], color="lightgrey")
    grid_x2 = grid_x + flow[..., 0]
    grid_y2 = grid_y + flow[..., 1]

    plot_grid(grid_x2[::step, ::step], grid_y2[::step, ::step], color="C0")
    plt.show()
if __name__ == "__main__":
    import os

    os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
    file1 = r'D:\codeflow\MPI-Sintel-complete\training\final\alley_1\frame_0006.png'
    file2 = r'D:\codeflow\MPI-Sintel-complete\training\final\alley_1\frame_0007.png'
    file_flow = r'D:\codeflow\MPI-Sintel-complete\training\flow\alley_1\frame_0006.flo'

    # file1 = r'D:\codeflow\FlyingChairs2\train\0000016-img_0.png'
    # file2 = r'D:\codeflow\FlyingChairs2\train\0000016-img_1.png'
    # file_flow = r'D:\codeflow\FlyingChairs2\train\0000016-flow_01.flo'
    im1 = cv2.imread(file1)[..., ::-1]
    h, w, c = im1.shape
    im2 = cv2.resize(cv2.imread(file2), (w, h))[..., ::-1]
    flow_gt = load_flow_to_numpy(file_flow)
    flow_gt = cv2.resize(flow_gt, (w, h))

    show_flow_im(im1, flow_gt)

    show_flow_grid(flow_gt, im1)

imagen:

12. Gráfico de flecha

def quiver_flow(flow):
    """
    flow : np.float32, [h, w, 2], (x, y) format
    """
    h, w = flow.shape[:2]
    xx, yy = np.meshgrid(range(w), range(h))

    s = 40
    plt.figure()
    plt.quiver(xx[::s, ::s], yy[::s, ::s], flow[..., 0][::s, ::s], flow[..., 1][::s, ::s], color='green')
    plt.show()