Preface
Taking into account the topological structure information of the road, this special feature, this blog post introduces two indicators Road Completeness and Road Connectivity to calculate the completeness and connectivity of road results.
Road Completeness
Lpred is the length of the predicted road, and Ltruth is the length of the road in the real label, where the prediction result of the road and the real label are both single-pixel raster data. Lpred is the length of the road surface area predicted by Ltruth through statistics.
The figure below shows how to calculate Road Completeness. The yellow is the predicted road, the blue is the real road, and the thin blue line is Ltruth after thinning. In the third picture, after superimposing, the green color falling on the predicted result represents the length of the successful prediction. , Namely Lpred, the red is the missing length.
import numpy as np
import skimage
from skimage import morphology
import cv2
def thin_image(mask_dir, mask_file):
im = cv2.imread(mask_dir + mask_file, 0)
im = im > 128
selem = skimage.morphology.disk(2)
im = skimage.morphology.binary_dilation(im, selem)
im = skimage.morphology.thin(im)
return im.astype(np.uint8) * 255
mask_dir = 'E:/shao_xing/out/result_boost12345/'
gt_dir = 'E:/shao_xing/test/lab/'
region = 'd'
mylogs = open('E:/shao_xing/out/tiny_evalog/new_metric/' + region + '_boost12345.log','w')
ratio_list=[]
for i in range(-4, 4):
for j in range(-4, 4):
gt_file = region + '_' + str(i) + '_' + str(j) + '_osm.png'
mask_file = gt_file[:-7] + 'merge.png'
mask = cv2.imread(mask_dir + mask_file, 0)
thin_gt = thin_image(gt_dir, gt_file)
num_mask = np.sum(thin_gt[mask > 128]) / 255
num_gt = np.sum(thin_gt) / 255
ratio = num_mask / (num_gt+0.00001)
if num_gt != 0:
ratio_list.append(ratio)
print('test image ', str(i), '_', str(j), 'ratio:', round(ratio, 2), file=mylogs)
print('test image ', str(i), '_', str(j), 'ratio:', round(ratio, 2))
print('********************************', file=mylogs)
print('Average Ratio:', round((sum(ratio_list) / len(ratio_list)), 2), file=mylogs)
print('********************************')
print('Average Ratio:', round((sum(ratio_list) / len(ratio_list)), 2))
mylogs.close()
Road Connectivity
The following formula represents the calculation method of Road Connectivity. Nconnected represents the number of consecutive road segments in the prediction result, and Ntotal represents the number of road segments in the true value. The statistical method is to divide the vector data of the true value into equal-length fragments, and then count whether these fragments are completely predicted. If they are completely predicted, they are recorded as connected, and if they are not completely predicted, they are recorded as unconnected.
The figure below shows the statistical process of Road Connectivity. The yellow is the predicted road, and the blue and red thin lines are the real road labels in vector format, which are composed of many fragments. The red one is the broken part, namely unconnected, the blue one is connected.
import sys
sys.path.append("./discoverlib")
from discoverlib import geom, graph
import numpy as np
import cv2
import skimage
from skimage import morphology
"""
evaluate connectivity based on ground truth graph
We split the total graph into segments with length of around 20 pixels
Then, statistic the number of fully connected segments in segmentation masks.
The connectivity ratio is the percentage of fully connected segments.
"""
log_name = 'mask' # evaluation log file
mask_dir = '~/data/out/mask/' # segmentation masks for evaluating
total_gt_number = 0
total_connected_number = 0
total_not_connected_number = 0
total_pred_number = 0
total_connected_length = 0
total_gt_length = 0
total_pred_length = 0
mylog = open('~/data/out/eval_log/' + log_name + '_connect.log', 'w')
region_name_list = [["amsterdam",-4,-4,4,4], ["chicago",-4,-4,4,4], ["denver",-4,-4,4,4]]
for region_info in region_name_list:
print("test region: "+region_info[0])
graph_name = '~/data/graph_gt/'+ region_info[0] + ".graph" # ground truth graph
gt_graph = graph.read_graph(graph_name)
edge_nodes=[]
for i,edge in enumerate(gt_graph.edges):
if i % 2 ==0:
edge_nodes.append([edge.src.point.x,edge.src.point.y,edge.dst.point.x,edge.dst.point.y])
base_gt_mask=np.zeros((1024, 1024))
edge_nodes=np.array(edge_nodes)
for i in range(region_info[1], region_info[3]):
for j in range(region_info[2], region_info[4]):
mask_file = region_info[0] + '_' + str(i) + '_' + str(j) + '_fusion.png'
# print(mask_dir + mask_file)
mask = cv2.imread(mask_dir + mask_file, 0)/255
patch_gt_number=0
patch_connected_number=0
patch_not_connected_number=0
patch_connected_length = 0
patch_gt_length = 0
offset=[-i*1024, -j*1024, -i*1024, -j*1024]
patch_nodes=edge_nodes+offset
for seg_edge in patch_nodes:
if (seg_edge>=[0,0,0,0]).all() and (seg_edge<[1024,1024,1024,1024]).all():
base_gt_mask = np.zeros((1024, 1024))
patch_gt_number+=1 # number of segments on the ground-truth graph
base_gt_mask=cv2.line(base_gt_mask,(seg_edge[0],seg_edge[1]),(seg_edge[2],seg_edge[3]),color=1, thickness=1)
pred_seg_length=np.sum(mask[base_gt_mask>0])
gt_length=np.sum(base_gt_mask>0)
patch_gt_length += gt_length
if pred_seg_length < gt_length:
patch_not_connected_number+=1
else:
patch_connected_number+=1
patch_connected_length += gt_length
else:
pass
im = (mask*255) > 128
selem = skimage.morphology.disk(2)
im = skimage.morphology.binary_dilation(im, selem)
im = skimage.morphology.thin(im)
thin_mask = im.astype(np.uint8) * 255
patch_pred_length = np.sum(thin_mask > 0)
patch_pred_number = patch_pred_length / 20.0 # number of segments on the prediction graph
ratio = 2*patch_connected_length/(patch_gt_length+patch_pred_length+0.00001)
print('test image {}_{} connected:not:total {}/{}/{}, ratio: {}'.format(i,j,patch_connected_number,
patch_not_connected_number,
patch_gt_number,
round(ratio, 4)))
print('test image {}_{} connected:not:total {}/{}/{}, ratio: {}'.format(i, j, patch_connected_number,
patch_not_connected_number,
patch_gt_number,
round(ratio, 4)), file=mylog)
total_gt_number += patch_gt_number
total_connected_number += patch_connected_number
total_not_connected_number += patch_not_connected_number
total_pred_number += patch_pred_number
total_connected_length += patch_connected_length
total_gt_length += patch_gt_length
total_pred_length += patch_pred_length
# total_ratio = 2*total_connected_number/(total_gt_number+total_pred_number)
total_ratio = 2*total_connected_length/(total_gt_length+total_pred_length)
print('********************************')
print("total connected:not:total {}/{}/{}, ratio: {}".format(total_connected_number,
total_not_connected_number,
total_gt_number,
round(total_ratio, 4)))
print("total_gt_length:{}".format(total_gt_length))
print("average gt length:{}".format(total_gt_length/total_gt_number))
print('********************************', file=mylog)
print("total connected:not:total {}/{}/{}, ratio: {}".format(total_connected_number,
total_not_connected_number,
total_gt_number,
round(total_ratio, 4)),
file=mylog)
print("total_gt_length:{}".format(total_gt_length),file=mylog)
print("average gt length:{}".format(total_gt_length/total_gt_number),file=mylog)
mylog.close()