yolov3训练日志可视化主要为loss和iou曲线的可视化，这些是我们查看训练效果的重要依据，首先看一个批次的日志输出：
说明：
一个批次有16*3条信息，每组包含三条信息，分别是：
Region 82 Avg IOU:
Region 94 Avg IOU:
Region 106 Avg IOU:
其中每行的参数意义如下：
Avg IOU:当前迭代中，预测的box与标注的box的平均交并比，越大越好，期望数值为1；
Class: 标注物体的分类准确率，越大越好，期望数值为1；
obj: 越大越好，期望数值为1；
No obj: 越小越好；
.5R: 以IOU=0.5为阈值时候的recall; recall = 检出的正样本/实际的正样本
0.75R: 以IOU=0.75为阈值时候的recall;
count:正样本数目。
注：存在nan值说明该子批次没有预测到正样本，在训练开始时候有出现是正常现象。
每个批次最后都会有一个输出行
1: 960.493347, 960.493347 avg, 0.000000 rate, 3.556313 seconds, 64 images各部分代表意义如下：
第几批次，总损失，平均损失，当前学习率，当前批次训练时间，目前为止参与训练的图片总数
#coding=utf-8
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import logging
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
logger = logging.getLogger(__name__)
class Yolov3LogVisualization:
def __init__(self,log_path,result_dir):
self.log_path = log_path
self.result_dir = result_dir
def extract_log(self, save_log_path, key_word):
with open(self.log_path, 'r') as f:
with open(save_log_path, 'w') as train_log:
next_skip = False
for line in f:
if next_skip:
next_skip = False
continue
# 去除多gpu的同步log
if 'Syncing' in line:
continue
# 去除除零错误的log
if 'nan' in line:
continue
if 'Saving weights to' in line:
next_skip = True
continue
if key_word in line:
train_log.write(line)
f.close()
train_log.close()
def parse_loss_log(self,log_path, line_num=2000):
result = pd.read_csv(log_path, skiprows=[x for x in range(line_num) if ((x % 10 != 9) | (x < 1000))],error_bad_lines=False, names=['loss', 'avg', 'rate', 'seconds', 'images'])
result['loss'] = result['loss'].str.split(' ').str.get(1)
result['avg'] = result['avg'].str.split(' ').str.get(1)
result['rate'] = result['rate'].str.split(' ').str.get(1)
result['seconds'] = result['seconds'].str.split(' ').str.get(1)
result['images'] = result['images'].str.split(' ').str.get(1)
result['loss'] = pd.to_numeric(result['loss'])
result['avg'] = pd.to_numeric(result['avg'])
result['rate'] = pd.to_numeric(result['rate'])
result['seconds'] = pd.to_numeric(result['seconds'])
result['images'] = pd.to_numeric(result['images'])
return result
def gene_loss_pic(self, pd_loss):
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(pd_loss['avg'].values, label='avg_loss')
ax.legend(loc='best')
ax.set_title('The loss curves')
ax.set_xlabel('batches')
fig.savefig(self.result_dir + '/avg_loss')
logger.info('save iou loss done')
def loss_pic(self):
train_log_loss_path = os.path.join(self.result_dir, 'train_log_loss.txt')
self.extract_log(train_log_loss_path, 'images')
pd_loss = self.parse_loss_log(train_log_loss_path)
self.gene_loss_pic(pd_loss)
def parse_iou_log(self,log_path, line_num=2000):
result = pd.read_csv(log_path, skiprows=[x for x in range(line_num) if (x % 10 == 0 or x % 10 == 9)],error_bad_lines=False,names=['Region Avg IOU', 'Class', 'Obj', 'No Obj', 'Avg Recall', 'count'])
result['Region Avg IOU'] = result['Region Avg IOU'].str.split(': ').str.get(1)
result['Class'] = result['Class'].str.split(': ').str.get(1)
result['Obj'] = result['Obj'].str.split(': ').str.get(1)
result['No Obj'] = result['No Obj'].str.split(': ').str.get(1)
result['Avg Recall'] = result['Avg Recall'].str.split(': ').str.get(1)
result['count'] = result['count'].str.split(': ').str.get(1)
result['Region Avg IOU'] = pd.to_numeric(result['Region Avg IOU'])
result['Class'] = pd.to_numeric(result['Class'])
result['Obj'] = pd.to_numeric(result['Obj'])
result['No Obj'] = pd.to_numeric(result['No Obj'])
result['Avg Recall'] = pd.to_numeric(result['Avg Recall'])
result['count'] = pd.to_numeric(result['count'])
return result
def gene_iou_pic(self, pd_loss):
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(pd_loss['Region Avg IOU'].values, label='Region Avg IOU')
# ax.plot(result['Class'].values,label='Class')
# ax.plot(result['Obj'].values,label='Obj')
# ax.plot(result['No Obj'].values,label='No Obj')
# ax.plot(result['Avg Recall'].values,label='Avg Recall')
# ax.plot(result['count'].values,label='count')
ax.legend(loc='best')
ax.set_title('The Region Avg IOU curves')
ax.set_xlabel('batches')
fig.savefig(self.result_dir + '/region_avg_iou')
logger.info('save iou pic done')
def iou_pic(self):
train_log_loss_path = os.path.join(self.result_dir, 'train_log_iou.txt')
self.extract_log(train_log_loss_path, 'IOU')
pd_loss = self.parse_iou_log(train_log_loss_path)
self.gene_iou_pic(pd_loss)
if __name__ == '__main__':
log_path = '/Users/songhongwei/Downloads/nohup.out'
result_dir = '/Users/songhongwei/PycharmProjects/py2project/hand/data'
logVis = Yolov3LogVisualization(log_path,result_dir)
logVis.loss_pic()
logVis.iou_pic()
效果：
