CV-1-目标检测-02-ver

import os
import glob
import numpy as np
import cv2
from skimage.feature import hog
import matplotlib.pyplot as plt
import numpy as np
import time
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from skimage.feature import hog
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
import pickle


# 1、read the car and notcar images-name list.
car_list = [name for index,name in enumerate(os.listdir('../vehicles/vehicles')) if index != 0]
noncar_list = [name for index,name in enumerate(os.listdir('../non-vehicles/non-vehicles')) if index != 0]

# load tha cars and notcars images.
cars=[]
for carfile in car_list:
    carpath='../vehicles/vehicles/'
    imgpath=os.path.join(carpath+carfile+'/*.png')
    #print(imgpath)
    images = glob.glob(imgpath)
    for image in images:
        cars.append(image)
print('车辆图片总数为:{}'.format(len(cars)))

noncars=[]
for carfile in noncar_list:
    carpath='../non-vehicles/non-vehicles/'
    imgpath=os.path.join(carpath+carfile+'/*.png')
    #print(imgpath)
    images=glob.glob(imgpath)
    for image in images:
        noncars.append(image)
print(len(noncars))


# todo 2、随机的可视化下图片Visualize the images randomly
def show_images():
    # random  choose car / not-car indices and plot example images
    car_ind = np.random.randint(0, len(cars))
    noncar_ind = np.random.randint(0, len(noncars))

    # Read in car / not-car images
    car_image = cv2.imread(cars[car_ind])
    noncar_image = cv2.imread(noncars[noncar_ind])
    print('car_image min Values is:', car_image.min(), 'Car_image Max values is :', car_image.max())
    print('notcar_image min Values is:', noncar_image.min(), 'notCar_image Max values is :', noncar_image.max())
    # Plot the examples
    plt.figure(figsize=(1, 1))
    fig = plt.figure()
    a = fig.add_subplot(121)
    plt.imshow(car_image)
    plt.title('Example Car Image')

    a = fig.add_subplot(122)
    plt.imshow(noncar_image)
    plt.title('Example Not-car Image')
    plt.show()

# ==============================================================
# 一、定义图片特征提取 并 训练模型
# ==============================================================

def color_hist(img, nbins=32, bins_range=(0, 256)):
    # 1、定义颜色直方图 HOG特征，并可视化
    # Define a function to return color_hist HOG features and visualization
    # 计算各个通道的颜色直方图。
    # Compute the histogram of the RGB channels separately
    fhist = np.histogram(img[:, :, 0], bins=nbins, range=bins_range)  # First channel histogram
    shist = np.histogram(img[:, :, 1], bins=nbins, range=bins_range)  # Second channel histogram
    thist = np.histogram(img[:, :, 2], bins=nbins, range=bins_range)  # Third channel histogram
    # Concatenate the histograms into a single feature vector
    hist_features = np.concatenate((fhist[0], shist[0], thist[0]))
    return hist_features


def bin_spatial(img, size=(32, 32)):
    # 2、直接将图片拉成向量，作为特征
    # Use cv2.resize().ravel() to create the feature vector
    features = cv2.resize(img, size).ravel()
    # Return the feature vector
    return features


def get_hog_features(img, orient, pix_per_cell, cell_per_block, vis=False, feature_vec=True):
    # 3、hog只能对单通道图片。 Call with two outputs if vis==True
    if vis == True:
        features, hog_image = hog(img, orientations=orient,
                                  pixels_per_cell=(pix_per_cell, pix_per_cell),
                                  cells_per_block=(cell_per_block, cell_per_block),
                                  transform_sqrt=True,
                                  visualize=vis,
                                  feature_vector=feature_vec)
        return features, hog_image
    # Otherwise call with one output
    else:
        features = hog(img, orientations=orient,
                       pixels_per_cell=(pix_per_cell, pix_per_cell),
                       cells_per_block=(cell_per_block, cell_per_block),
                       transform_sqrt=True,
                       visualize=vis,
                       feature_vector=feature_vec)
        return features



def extract_features(imgs, color_space='HSV', spatial_size=(16, 16), hist_bins=32,
                     orient=11, pix_per_cell=16, cell_per_block=2, hog_channel='ALL',
                     spatial_feat=True, hist_feat=True, hog_feat=True):
    """
    输入：一个图片的列表，调用bin_spatial() 和 color_hist()  get_hog_features()提取特征向量
    :param imgs:
    :param color_space:
    :param spatial_size:
    :param hist_bins:
    :param orient:
    :param pix_per_cell:
    :param cell_per_block:
    :param hog_channel:
    :param spatial_feat:
    :param hist_feat:
    :param hog_feat:
    :return:
    """
    # Create a list to append feature vectors to
    features = []
    # Iterate through the list of images
    for file in imgs:
        file_features = []
        # Read in each one by one
        image = cv2.imread(file)
        # apply color conversion if other than 'RGB'
        if color_space != 'BGR':
            if color_space == 'HSV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
            elif color_space == 'LUV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2LUV)
            elif color_space == 'HLS':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2HLS)
            elif color_space == 'YUV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2YUV)
            elif color_space == 'YCrCb':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2YCrCb)
        else:
            feature_image = np.copy(image)

        if spatial_feat == True:
            spatial_features = bin_spatial(feature_image, size=spatial_size)
            file_features.append(spatial_features)
        if hist_feat == True:
            # Apply color_hist()
            hist_features = color_hist(feature_image, nbins=hist_bins)
            file_features.append(hist_features)
        if hog_feat == True:
            # Call get_hog_features() with vis=False, feature_vec=True
            if hog_channel == 'ALL':  # 代表所有颜色通道都提取hog特征
                hog_features = []
                for channel in range(feature_image.shape[2]):
                    hog_feature = get_hog_features(feature_image[:, :, channel],
                                                   orient, pix_per_cell, cell_per_block,
                                                   vis=False, feature_vec=True)
                    # print('hog_feature shape is:{},type is{}'.format(hog_feature.shape,type(hog_feature)))
                    hog_features.append(hog_feature)
                hog_features = np.ravel(hog_features)
            else:
                hog_features = get_hog_features(feature_image[:, :, hog_channel], orient,
                                                pix_per_cell, cell_per_block, vis=False, feature_vec=True)
            # Append the new feature vector to the features list
            file_features.append(hog_features)
        features.append(np.concatenate(file_features))
    # Return list of feature vectors
    return features


# 4、提取直方图特征 并且可视化
def show_hog():
    pix_per_cell = [8, 16, 8, 16]
    cell_per_block = [1, 1, 2, 2]
    orient = [9, 9, 11, 11]

    for i in range(len(pix_per_cell)):
        # Input is an image of single channel or gray
        car_ind = 1167
        car_image = cv2.imread(cars[car_ind])

        gray = cv2.cvtColor(car_image, cv2.COLOR_BGR2GRAY)
        features, hog_image = get_hog_features(
            gray, orient[i], pix_per_cell[i], cell_per_block[i], vis=True, feature_vec=True
        )

        # Plot the examples
        plt.figure(figsize=(1, 1))
        fig = plt.figure()
        a = fig.add_subplot(121)
        plt.imshow(car_image, cmap='gray')
        plt.title('Example Car Image')

        a = fig.add_subplot(122)
        plt.imshow(hog_image, cmap='gray')
        plt.title('HOG Visualization')

        print(" No. of features for pix_per_cell- ", pix_per_cell[i], "cell_per_block -", cell_per_block[i],
              'orient -', orient[i], "is", len(features))
    plt.show()

# show_hog()


def train_svm_model():
    # train the  SVM classifier model
    # 1、定义模型超参数
    color_space = 'YUV'  # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
    orient = 11          # HOG orientations
    pix_per_cell = 16    # HOG pixels per cell
    cell_per_block = 2   # HOG cells per block
    hog_channel = 'ALL'  # Can be 0, 1, 2, or "ALL"

    car_features = extract_features(
        cars, color_space=color_space, spatial_size=(16, 16), hist_bins=32, orient=11,
        pix_per_cell=16, cell_per_block=2, hog_channel=hog_channel)

    print('car_features length is:{},第一个feature shape is:{}'.format(len(car_features), car_features[0].shape))

    notcar_features = extract_features(
        noncars, color_space=color_space, spatial_size=(16, 16), hist_bins=32, orient=11,
        pix_per_cell=16, cell_per_block=2, hog_channel=hog_channel)

    print('not_car_features length is:{},第一个feature shape is:{}'.format(len(notcar_features), notcar_features[0].shape))

    # Create an array stack of feature vectors
    X = np.vstack((car_features, notcar_features)).astype(np.float64)
    print('X shape is:{},X min is {},X max is {}'.format(X.shape, X.min(), X.max()))

    #  fixme 定义类别的标签（即车辆为1，背景为0）
    y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))

    # 将数据随机的分成 训练 和 测试数据集
    rand_state = np.random.randint(0, 100)
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.20, random_state=rand_state)
    print('X_train min value is {},X_train max value is {}'.format(X_train.min(), X_train.max()))
    print('X_test min value is {},X_test max value is {}'.format(X_test.min(), X_test.max()))

    # Fit a per-column scaler
    X_scaler = StandardScaler().fit(X_train)
    # Apply the scaler to X
    X_train = X_scaler.transform(X_train)
    X_test = X_scaler.transform(X_test)

    print('---After StandardScaler---  X_train min value is {},X_train max value is {},Mean is {}'.
          format(X_train.min(), X_train.max(), X_train.mean()))
    print('---After StandardScaler---  X_test min value is {},X_test max value is {},Mean is {}'.
          format(X_test.min(), X_test.max(), X_train.mean()))

    print('Using:', orient, 'orientations', pix_per_cell,
          'pixels per cell and', cell_per_block, 'cells per block')
    print('Feature vector length:', len(X_train[0]))
    # Use a linear SVC
    svc = LinearSVC()
    # Check the training time for the SVC
    t = time.time()
    svc.fit(X_train, y_train)
    t2 = time.time()
    print(round(t2 - t, 2), 'Seconds to train SVC...')
    # Check the score of the SVC
    print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))

    # Save the svc and X_scaler
    dist_pickle = {}
    dist_pickle["svc"] = svc
    dist_pickle["X_scaler"] = X_scaler
    pickle.dump(dist_pickle, open("../svc_pickle1.p", "wb"))
    print('模型成功保存!!')

# train_svm_model()


# =================================================================== #
# 二、模型训练完毕，先到图片上预测
# =================================================================== #

def convert_color(img, conv='BGR2YCrCb'):
    # Define function to find cars
    if conv == 'BGR2YCrCb':
        return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
    elif conv == 'BGR2LUV':
        return cv2.cvtColor(img, cv2.COLOR_BGR2LUV)
    elif conv == 'BGR2HSV':
        return cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    elif conv == 'BGR2HLS':
        return cv2.cvtColor(img, cv2.COLOR_BGR2HLS)
    elif conv == 'BGR2YUV':
        return cv2.cvtColor(img, cv2.COLOR_BGR2YUV)


# Define a single function that can extract features using hog sub-sampling and make predictions
def find_cars(img, ystart, ystop, scale, svc, X_scaler, spatial_size, hist_bins, orient, pix_per_cell,
              cell_per_block, conv='BGR2YUV'):
    draw_img = np.copy(img)
    heatmap = np.zeros_like(img[:, :, 0])  # 定义一个热力图背景, 初始值全为0

    img_tosearch = img[ystart:ystop, :, :]  # fixme 缩小图片搜索的范围，直接依据经验切片
    # convert color space function
    ctrans_tosearch = convert_color(img_tosearch, conv=conv)

    # scale=the origin scaled ratio fixme 再次按照缩放比例缩小待检测的图片（缩小了1.5倍）
    if scale != 1:
        imshape = ctrans_tosearch.shape
        ctrans_tosearch = cv2.resize(
            ctrans_tosearch, (np.int(imshape[1] / scale), np.int(imshape[0] / scale)))

    ch1 = ctrans_tosearch[:, :, 0]
    ch2 = ctrans_tosearch[:, :, 1]
    ch3 = ctrans_tosearch[:, :, 2]

    # Define blocks and steps as above
    nxblocks = (ch1.shape[1] // pix_per_cell) - 1
    nyblocks = (ch1.shape[0] // pix_per_cell) - 1
    nfeat_per_block = orient * cell_per_block ** 2

    # 64 was the orginal sampling rate, with 8 cells and 16 pix per cell
    window = 64
    nblocks_per_window = (window // pix_per_cell) - 1
    cells_per_step = 2  # Instead of overlap, define how many cells to step 2
    nxsteps = (nxblocks - nblocks_per_window) // cells_per_step
    nysteps = (nyblocks - nblocks_per_window) // cells_per_step

    # Compute individual channel HOG features for the entire image 对待检测的整个图提取了HOG特征。
    hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
    # define the bbox_list for detect False positive and heat map
    bbox_list = []
    for xb in range(nxsteps):
        for yb in range(nysteps):
            ypos = yb * cells_per_step
            xpos = xb * cells_per_step
            # Extract HOG for this patch
            hog_feat1 = hog1[ypos:ypos + nblocks_per_window, xpos:xpos + nblocks_per_window].ravel()
            hog_feat2 = hog2[ypos:ypos + nblocks_per_window, xpos:xpos + nblocks_per_window].ravel()
            hog_feat3 = hog3[ypos:ypos + nblocks_per_window, xpos:xpos + nblocks_per_window].ravel()
            hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))

            xleft = xpos * pix_per_cell
            ytop = ypos * pix_per_cell

            # Extract the image patch
            subimg = cv2.resize(ctrans_tosearch[ytop:ytop + window, xleft:xleft + window], (64, 64))

            spatial_features = bin_spatial(subimg, size=spatial_size)
            hist_features = color_hist(subimg, nbins=hist_bins)

            # 标准化特征并做预测。 Scale features and make a prediction
            test_features = X_scaler.transform(
                np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))
            test_prediction = svc.predict(test_features)

            if test_prediction == 1:  # 即预测该区域有小车
                xbox_left = np.int(xleft * scale)
                ytop_draw = np.int(ytop * scale)
                win_draw = np.int(window * scale)
                bbox_list.append(((xbox_left, ytop_draw + ystart),
                                  (xbox_left + win_draw, ytop_draw + win_draw + ystart)))
                cv2.rectangle(draw_img, (xbox_left, ytop_draw + ystart),
                              (xbox_left + win_draw, ytop_draw + win_draw + ystart), (0, 0, 255), 6)
                heatmap[ytop_draw + ystart:ytop_draw + win_draw + ystart, xbox_left:xbox_left + win_draw] += 1

    return draw_img, heatmap, bbox_list


def test_images():
    # 定义超参数
    spatial_size = (16, 16)
    hist_bins = 32
    ystart = 400
    ystop = 660
    orient = 11
    pix_per_cell = 16
    cell_per_block = 2
    scale = 1.5

    images = glob.glob('../test_images/test*.jpg')  # 遍历读取图片的路径

    # 读取预训练的模型
    f = open("../svc_pickle.p", "rb")
    dist_pickle = pickle.load(f)
    svc = dist_pickle['svc']
    X_scaler = dist_pickle["X_scaler"]

    for fname in images:
        img = cv2.imread(fname)
        draw_img, heatmap, bbox_list = find_cars(
            img, ystart, ystop, scale, svc, X_scaler, spatial_size, hist_bins, orient,
            pix_per_cell, cell_per_block, conv='BGR2YUV')

        fig = plt.figure(figsize=(10, 10))
        plt.subplot(121)
        plt.imshow(draw_img)
        plt.title('Draw box image')

        plt.subplot(122)
        plt.imshow(heatmap, cmap='hot')
        plt.title('Hotimage')
        fig.tight_layout()
    plt.show()

# test_images()


# =================================================================== #
# 三、Multiple Detections & False Positives   实现NMS
# =================================================================== #

from scipy.ndimage.measurements import label
def add_heat(heatmap, bbox_list):
    # Iterate through list of bboxes
    for box in bbox_list:
        # Add += 1 for all pixels inside each bbox
        # Assuming each "box" takes the form ((x1, y1), (x2, y2))
        heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1
    return heatmap  # Iterate through list of bboxes


def apply_threshold(heatmap, threshold):
    # Zero out pixels below the threshold
    heatmap[heatmap <= threshold] = 0
    # Return thresholded map
    return heatmap


def draw_labeled_bboxes(img, labels):
    # Iterate through all detected cars
    for car_number in range(1, labels[1] + 1):
        # Find pixels with each car_number label value
        nonzero = (labels[0] == car_number).nonzero()
        # Identify x and y values of those pixels
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Define a bounding box based on min/max x and y
        bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
        # Draw the box on the image
        cv2.rectangle(img, bbox[0], bbox[1], (0, 0, 255), 6)
    # Return the image
    return img


def detect_false_and_applyto_origin(img, box_list, threshold=1, draw_pic=False):
    heat = np.zeros_like(img[:, :, 0]).astype(np.float)
    # Add heat to each box in box list
    heat = add_heat(heat, box_list)

    # Apply threshold to help remove false positives
    heat = apply_threshold(heat, threshold)

    # Visualize the heatmap when displaying
    heatmap = np.clip(heat, 0, 255)

    # Find final boxes from heatmap using label function
    labels = label(heatmap)
    draw_img = draw_labeled_bboxes(np.copy(img), labels)

    if draw_pic == True:
        plt.figure(figsize=(10, 10))
        fig = plt.figure()

        plt.imshow(draw_img)
        plt.show()
    return draw_img


def test1():
    spatial_size = (16, 16)
    hist_bins = 32
    ystart = 400
    ystop = 660
    orient = 11
    pix_per_cell = 16
    cell_per_block = 2
    scale = 1.5
    images = glob.glob('../test_images/test*.jpg')

    # 读取预训练的模型
    f = open("../svc_pickle.p", "rb")
    dist_pickle = pickle.load(f)
    svc = dist_pickle['svc']
    X_scaler = dist_pickle["X_scaler"]

    for fname in images:
        img = cv2.imread(fname)
        draw_img, heatmap, bbox_list = find_cars(
            img, ystart, ystop, scale, svc, X_scaler, spatial_size, hist_bins, orient,
            pix_per_cell, cell_per_block, conv='BGR2YUV')
        result = detect_false_and_applyto_origin(img, bbox_list, threshold=0.8, draw_pic=True)
    plt.show()

# test1()

def process_image(img, n_frames=20, thresholding=18):
    spatial_size = (16, 16)
    hist_bins = 32
    ystart = 400
    ystop = 660
    orient = 11
    pix_per_cell = 16
    cell_per_block = 2
    scale = 1.5

    f = open("../svc_pickle.p", "rb")
    dist_pickle = pickle.load(f)
    svc = dist_pickle['svc']
    X_scaler = dist_pickle["X_scaler"]

    bbox_list_temp = []
    global hot_windows_list
    global bbox_list
    global bbox_list_final
    draw_img, heatmap, bbox_list = find_cars(img, ystart, ystop, scale, svc, X_scaler, spatial_size, hist_bins, orient,
                                             pix_per_cell, cell_per_block, conv='BGR2YUV')

    hot_windows_list.append(bbox_list)

    if len(hot_windows_list) <= n_frames:
        bbox_list_final = sum(hot_windows_list, [])  # Add windows from all available frames

    # Look at last n frames and append all hot windows found
    else:
        for val in hot_windows_list[(len(hot_windows_list) - n_frames - 1): (len(hot_windows_list) - 1)]:
            bbox_list_temp.append(val)
        # Flatten this list
        bbox_list_final = sum(bbox_list_temp, [])

    result = detect_false_and_applyto_origin(
        img, bbox_list_final, threshold=thresholding, draw_pic=False)
    return result

hot_windows_list = []

from moviepy.editor import VideoFileClip
import pickle

print('开始对视频进行预测!')
video = VideoFileClip("../project_video.mp4")
project_clip = video.fl_image(process_image)  # fixme NOTE: this function expects color images!!
output = "../vehicle_detection_bei1.mp4"
project_clip.write_videofile(output, audio=False)
print('对视频预测结束!')


"""
传统视觉问题：
    1、slide windows 太慢了，而且窗口太难设置了；
    2、Hog提取特征能力有限，不断尝试超参数；
    3、预测也非常慢。
"""