Mutual conversion between data set annotation files when training different models of the yolo series

Mutual conversion between data set annotation files when training different models of the yolo series

1. Convert the json file marked by the annotation wizard into the txt format required for yolov5 training. Each json file needs to be traversed in turn.

def json2txt():
    from tqdm import tqdm, trange

    ''' 标注精灵的json（x1,y1,x2,y2）文件转训练所用txt（x0,y0,w,h）'''
    
    name2307 = {'pedes': 0, 'car': 1, 'bus': 2, 'bigtru': 3, 'bike': 4, 'elec': 5, 'tricycle': 6, 'coni': 7, 'warm': 8,
               'polic': 1, 'tralight': 9, 'ambu': 10, 'fireen': 10,
              'pedes            ': 0, 'car ': 1, '  car': 1, ' car': 1, ' ca   r': 1, 'suv': 1, 'van': 1,
              'mpv': 1, 'pickup': 1, 'ar': 1, 'pedesd': 0, 'coach ': 2, 'car  ': 1,
              'suv ': 1, 'coach': 2,'trailer': 3, 'crane': 3, 'smatru': 3, 'tank': 3, 'escort': 10,
              'engine': 10, 'bull': 10, 'excava': 10, 'military': 10, 'elec ': 5, 'moto': 5, 'motopoli': 5}
    #name2307 = {#pedes: 0,#car: 1,#bus: 2,#truck: 3,#bike: 4,#elec: 5,#tricycle: 6,#coni: 7,#warm: 8,#tralight: 9,#special_vehicles: 10,}    

    # 如下参数需根据情况进行修改
    namell = name2307
    pathimg =    r'D:\yolov5train\datasTrain3_More\images\val'
    pathjson = r'E:\datasTrainALL\OrdinaryRoad00_outputs44285'
    despathtxt = r'D:\yolov5train\datasTrain3_More\labels\val'

    files = os.listdir(pathimg)
    print(len(files))
    # files = [x for x in files if x[-4:] == '.jpg']
    ic = 0
    ycfile = []
    # for f in files[:]:
    for i in tqdm(range(len(files)), desc='进度：'):
        f = files[i]
        ic = ic + 1
        # print(ic, ':  ', f)

        if (f[-4:] != '.jpg'):
            continue
        imgShape = cv2.imread(os.path.join(pathimg, f)).shape
        imgw, imgh = imgShape[1], imgShape[0]

        json_path_name = os.path.join(pathjson, f[:-4] + '.json')
        with open(json_path_name, 'r', encoding='utf-8') as rf:
            rf = json.load(rf)

            try:
                objs = rf['outputs']['object']
            except:
                ycfile.append(f)
                print(ycfile)
                # break
                continue
            boxs = []
            for o in objs:
                try:
                    id = o['name']
                    if id not in namell.keys():
                        continue
                    id = str(namell[id])
                    xmin = o['bndbox']['xmin']
                    ymin = o['bndbox']['ymin']
                    xmax = o['bndbox']['xmax']
                    ymax = o['bndbox']['ymax']
                    ymax = imgh if (ymax > imgh or (imgh - ymax<5)) else ymax
                    ymin = 0 if  ymin < 5 else ymin
                    xmax = imgw if (xmax > imgw or (imgw - xmax < 5)) else xmax
                    xmin = 0 if xmin < 5 else xmin
                    x = (xmin + xmax) / 2 / imgw
                    y = (ymin + ymax) / 2 / imgh
                    w = (xmax - xmin) / imgw
                    h = (ymax - ymin) / imgh
                    x = str(x) if len(str(x)) <= 7 else str(x)[:7]
                    y = str(y) if len(str(y)) <= 7 else str(y)[:7]
                    w = str(w) if len(str(w)) <= 7 else str(w)[:7]
                    h = str(h) if len(str(h)) <= 7 else str(h)[:7]
                    box = id + ' ' + x + ' ' + y + ' ' + w + ' ' + h + '\n'
                    boxs.append(box)
                except:
                    continue
            if boxs == []:
                continue
            boxs[-1] = boxs[-1].replace('\n', '')

        txt_path_name = os.path.join(despathtxt, f[:-4] + '.txt')
        with open(txt_path_name, 'w', encoding='utf-8') as wf:
            wf.writelines(boxs)

    print(ycfile)

2. To convert the txt format of yolov5 into the xml of yolovX, each txt file needs to be traversed in sequence.

ref :https://zhuanlan.zhihu.com/p/525950939

def txt2xml():
    """
    for YOLOX
    将yolov5的txt格式转化为yolovX的xml
    """
    # ref :https://zhuanlan.zhihu.com/p/525950939

    from xml.dom.minidom import Document
    import os
    import cv2

    """
    此函数用于将yolov5格式txt标注文件转换为voc格式xml标注文件
    在自己的标注图片文件夹下建三个子文件夹
    """
    dic = {'0': "peses",  # 创建字典用来对类型进行转换
           '1': "car",  # 此处的字典要与自己的classes.txt文件中的类对应，且顺序要一致
           '2': "bus",
           '3': "truck",
           '4': "bike",
           '5': "elec",
           '6': "tricycle",
           '7': "coni",
           '8': "warm",
           '9': "tralight",
           '10': "special_vehicle"
           }
    picPath = "D:/yolov5train/datasTrain3_More/images/train/"  # 图片所在文件夹路径，后面的/一定要带上
    txtPath = "D:/yolov5train/datasTrain3_More/labels/train/"  # txt所在文件夹路径，后面的/一定要带上
    xmlPath = "D:/yolov5train/datasTrain3_More/annotations/train/"  # xml文件保存路径，后面的/一定要带上
    files = os.listdir(txtPath)
    for i, name in enumerate(files):
        xmlBuilder = Document()
        annotation = xmlBuilder.createElement("annotation")  # 创建annotation标签
        xmlBuilder.appendChild(annotation)
        txtFile = open(txtPath + name)
        txtList = txtFile.readlines()
        img = cv2.imread(picPath + name[0:-4] + ".jpg")
        Pheight, Pwidth, Pdepth = img.shape

        folder = xmlBuilder.createElement("folder")  # folder标签
        foldercontent = xmlBuilder.createTextNode("driving_annotation_dataset")
        folder.appendChild(foldercontent)
        annotation.appendChild(folder)  # folder标签结束

        filename = xmlBuilder.createElement("filename")  # filename标签
        filenamecontent = xmlBuilder.createTextNode(name[0:-4] + ".jpg")
        filename.appendChild(filenamecontent)
        annotation.appendChild(filename)  # filename标签结束

        size = xmlBuilder.createElement("size")  # size标签
        width = xmlBuilder.createElement("width")  # size子标签width
        widthcontent = xmlBuilder.createTextNode(str(Pwidth))
        width.appendChild(widthcontent)
        size.appendChild(width)  # size子标签width结束

        height = xmlBuilder.createElement("height")  # size子标签height
        heightcontent = xmlBuilder.createTextNode(str(Pheight))
        height.appendChild(heightcontent)
        size.appendChild(height)  # size子标签height结束

        depth = xmlBuilder.createElement("depth")  # size子标签depth
        depthcontent = xmlBuilder.createTextNode(str(Pdepth))
        depth.appendChild(depthcontent)
        size.appendChild(depth)  # size子标签depth结束

        annotation.appendChild(size)  # size标签结束

        for j in txtList:
            oneline = j.strip().split(" ")
            object = xmlBuilder.createElement("object")  # object 标签
            picname = xmlBuilder.createElement("name")  # name标签
            namecontent = xmlBuilder.createTextNode(dic[oneline[0]])
            picname.appendChild(namecontent)
            object.appendChild(picname)  # name标签结束

            pose = xmlBuilder.createElement("pose")  # pose标签
            posecontent = xmlBuilder.createTextNode("Unspecified")
            pose.appendChild(posecontent)
            object.appendChild(pose)  # pose标签结束

            truncated = xmlBuilder.createElement("truncated")  # truncated标签
            truncatedContent = xmlBuilder.createTextNode("0")
            truncated.appendChild(truncatedContent)
            object.appendChild(truncated)  # truncated标签结束

            difficult = xmlBuilder.createElement("difficult")  # difficult标签
            difficultcontent = xmlBuilder.createTextNode("0")
            difficult.appendChild(difficultcontent)
            object.appendChild(difficult)  # difficult标签结束

            bndbox = xmlBuilder.createElement("bndbox")  # bndbox标签
            xmin = xmlBuilder.createElement("xmin")  # xmin标签
            mathData = int(((float(oneline[1])) * Pwidth + 1) - (float(oneline[3])) * 0.5 * Pwidth)
            xminContent = xmlBuilder.createTextNode(str(mathData))
            xmin.appendChild(xminContent)
            bndbox.appendChild(xmin)  # xmin标签结束

            ymin = xmlBuilder.createElement("ymin")  # ymin标签
            mathData = int(((float(oneline[2])) * Pheight + 1) - (float(oneline[4])) * 0.5 * Pheight)
            yminContent = xmlBuilder.createTextNode(str(mathData))
            ymin.appendChild(yminContent)
            bndbox.appendChild(ymin)  # ymin标签结束

            xmax = xmlBuilder.createElement("xmax")  # xmax标签
            mathData = int(((float(oneline[1])) * Pwidth + 1) + (float(oneline[3])) * 0.5 * Pwidth)
            xmaxContent = xmlBuilder.createTextNode(str(mathData))
            xmax.appendChild(xmaxContent)
            bndbox.appendChild(xmax)  # xmax标签结束

            ymax = xmlBuilder.createElement("ymax")  # ymax标签
            mathData = int(((float(oneline[2])) * Pheight + 1) + (float(oneline[4])) * 0.5 * Pheight)
            ymaxContent = xmlBuilder.createTextNode(str(mathData))
            ymax.appendChild(ymaxContent)
            bndbox.appendChild(ymax)  # ymax标签结束

            object.appendChild(bndbox)  # bndbox标签结束

            annotation.appendChild(object)  # object标签结束

        f = open(xmlPath + name[0:-4] + ".xml", 'w')
        xmlBuilder.writexml(f, indent='\t', newl='\n', addindent='\t', encoding='utf-8')
        f.close()

        # break


def ImageSetsMain():
    """
    for YOLOX：将每一个数据文件名称保存在txt文件中
    """
    imgPath = r'D:\yolov5train\datasTrain3_More\images\train'
    xmlPath = r'D:\yolov5train\datasTrain3_More\labels\train'
    train_val_txtPath = r'D:\yolov5train\datasTrain3_More\ImageSets\Main\train.txt'

    imgs = os.listdir(imgPath)
    with open(train_val_txtPath, 'w') as wf:
        for i in range(len(imgs)):
            text = imgs[i].split(".jpg")[0] +'\n'
            wf.write(text)

3. Convert the yolo format annotation file .txt to the coco format annotation file .json

import argparse
import json
import os
import sys
import shutil
from datetime import datetime

import cv2

coco = dict()
coco['images'] = []
coco['type'] = 'instances'
coco['annotations'] = []
coco['categories'] = []

category_set = dict()
image_set = set()

image_id = 000000
annotation_id = 0


def addCatItem(category_dict):
    for k, v in category_dict.items():
        category_item = dict()
        category_item['supercategory'] = 'none'
        category_item['id'] = int(k)
        category_item['name'] = v
        coco['categories'].append(category_item)


def addImgItem(file_name, size):
    global image_id
    image_id += 1
    image_item = dict()
    image_item['id'] = image_id
    image_item['file_name'] = file_name
    image_item['width'] = size[1]
    image_item['height'] = size[0]
    image_item['license'] = None
    image_item['flickr_url'] = None
    image_item['coco_url'] = None
    image_item['date_captured'] = str(datetime.today())
    coco['images'].append(image_item)
    image_set.add(file_name)
    return image_id


def addAnnoItem(object_name, image_id, category_id, bbox):
    global annotation_id
    annotation_item = dict()
    annotation_item['segmentation'] = []
    seg = []
    # bbox[] is x,y,w,h
    # left_top
    seg.append(bbox[0])
    seg.append(bbox[1])
    # left_bottom
    seg.append(bbox[0])
    seg.append(bbox[1] + bbox[3])
    # right_bottom
    seg.append(bbox[0] + bbox[2])
    seg.append(bbox[1] + bbox[3])
    # right_top
    seg.append(bbox[0] + bbox[2])
    seg.append(bbox[1])

    annotation_item['segmentation'].append(seg)

    annotation_item['area'] = bbox[2] * bbox[3]
    annotation_item['iscrowd'] = 0
    annotation_item['ignore'] = 0
    annotation_item['image_id'] = image_id
    annotation_item['bbox'] = bbox
    annotation_item['category_id'] = category_id
    annotation_id += 1
    annotation_item['id'] = annotation_id
    coco['annotations'].append(annotation_item)


def xywhn2xywh(bbox, size):
    bbox = list(map(float, bbox))
    size = list(map(float, size))
    xmin = (bbox[0] - bbox[2] / 2.) * size[1]
    ymin = (bbox[1] - bbox[3] / 2.) * size[0]
    w = bbox[2] * size[1]
    h = bbox[3] * size[0]
    box = (xmin, ymin, w, h)
    return list(map(int, box))


def parseXmlFilse(image_path, anno_path, save_path, json_name='train.json'):
    assert os.path.exists(image_path), "ERROR {} dose not exists".format(image_path)
    assert os.path.exists(anno_path), "ERROR {} dose not exists".format(anno_path)
    if os.path.exists(save_path):
        pass
        # shutil.rmtree(save_path)
    os.makedirs(save_path)
    json_path = os.path.join(save_path, json_name)

    category_set = []
    with open(anno_path + '/classes.txt', 'r') as f:
        for i in f.readlines():
            category_set.append(i.strip())
    category_id = dict((k, v) for k, v in enumerate(category_set))
    addCatItem(category_id)

    images = [os.path.join(image_path, i) for i in os.listdir(image_path)]
    files = [os.path.join(anno_path, i) for i in os.listdir(anno_path)]
    images_index = dict((v.split(os.sep)[-1][:-4], k) for k, v in enumerate(images))
    for file in files:
        if os.path.splitext(file)[-1] != '.txt' or 'classes' in file.split(os.sep)[-1]:
            continue
        if file.split(os.sep)[-1][:-4] in images_index:
            index = images_index[file.split(os.sep)[-1][:-4]]
            img = cv2.imread(images[index])
            shape = img.shape
            filename = images[index].split(os.sep)[-1]
            current_image_id = addImgItem(filename, shape)

        else:
            continue
        with open(file, 'r') as fid:
            for i in fid.readlines():
                i = i.strip().split()
                category = int(i[0])
                category_name = category_id[category]
                bbox = xywhn2xywh((i[1], i[2], i[3], i[4]), shape)
                addAnnoItem(category_name, current_image_id, category, bbox)

    json.dump(coco, open(json_path, 'w'))
    print("class nums:{}".format(len(coco['categories'])))
    print("image nums:{}".format(len(coco['images'])))
    print("bbox nums:{}".format(len(coco['annotations'])))





def MultTxt2SingleJson():
    """
        脚本说明：
            本脚本用于将yolo格式的标注文件.txt转换为coco格式的标注文件.json
        参数说明：
            anno_path:标注文件txt存储路径
            save_path:json文件输出的文件夹
            image_path:图片路径
            json_name:json文件名字
        """
    parser = argparse.ArgumentParser()
    parser.add_argument('-ap', '--anno-path', type=str, default=r'D:\yolov5train\datasTrain3_More\labels\val', help='yolo txt path')
    parser.add_argument('-s', '--save-path', type=str, default=r'D:\yolov5train\datasTrain3_More\images\Annotations', help='json save path')
    parser.add_argument('--image-path', default=r'D:\yolov5train\datasTrain3_More\images\val')
    parser.add_argument('--json-name', default='val.json')

    opt = parser.parse_args()
    if len(sys.argv) > 1:
        print(opt)
        parseXmlFilse(**vars(opt))
    else:
        anno_path = r'D:\yolov5train\datasTrain3_More\labels\val'
        save_path = r'D:\yolov5train\datasTrain3_More\images\Annotations'
        image_path = r'D:\yolov5train\datasTrain3_More\images\val'
        json_name = 'val.json'
        parseXmlFilse(image_path, anno_path, save_path, json_name)
    pass

4. Write the paths to the txt files to store the training set and validation set images respectively.

def writeImgPath_forYolov7Datas():
    '''
    因为yolov7训练的时候，在数据集根目录下，需要有txt文件分别存放训练集和验证集图片的路径
    '''
    from tqdm import tqdm
    import os
    imgpath = r'D:/yolov5train/datasTrain3_More/images/train'
    txt_path_name = r'D:/yolov5train/datasTrain3_More/train_list.txt'
    files = os.listdir(imgpath)
    imgspath = []
    for i in tqdm(range(len(files)), desc='进度 '):
        f = files[i]
        try:
            name = os.path.join(imgpath, f)
            name = name + '\n'
            imgspath.append(name)
        except Exception as e:
            print(f, ":", e)
            continue
    imgspath[-1] = imgspath[-1].replace('\n', '')
    with open(txt_path_name, 'w', encoding='utf-8') as wf:
        wf.writelines(imgspath)

5. Convert labelme’s json to yolo’s txt

def labelmejson2yolotxt():
    import os
    import numpy as np
    from tqdm import tqdm

    imgPath = r' '
    jsonPath = r' '
    savetxtPath = r' '
    clsStatic, clsStaticFlag= {
    
    }, True
    # 先统计标签有哪些再保存txt，根据统计的标签 好确定 namell
    savetxtPath, savetxtPathFlag = savetxtPath, True
    # name15label = [pedes0, car1, bus2, truck3, bike4, elec5, tricycle6, coni7, warm8, tralight9, polic10, ambu11,
    #                fireen12, suv13, mpv14]
    name15 = {
    
    'car': 1, 'suv': 13, 'mpv': 14, 'van': 14, 'polic': 10, 'smatru': 3, 'elec': 5,
              'pedes': 0, 'coni': 7, 'tank': 3, 'moto': 5, 'bigtru': 3, 'bus': 2, 'engine': 3,
              'tricycle': 6, 'pickup': 1, 'coach': 2, 'bike': 4, 'pedesr': 0, 'crane': 3, 'trailer': 3,
              'excava': 3, 'warm': 8, 'tralight': 9, 'truck': 3, 'bigtrur': 3, 'vcar': 1, 'escort': 3 }
    namell =  name15


    files = os.listdir(imgPath)
    print('len(imgPath): ', len(files))
    # files = [x for x in files if x[-4:] == '.jpg']

    for i in tqdm(range(len(files)), desc='进度：'):
        f = files[i]

        if (f[-4:] != '.jpg'):
            continue
        imgShape = cv2.imread(os.path.join(imgPath, f)).shape
        imgw, imgh = imgShape[1], imgShape[0]

        json_path_name = os.path.join(jsonPath, f[:-4] + '.json')
        with open(json_path_name, 'r', encoding='utf-8') as rf:
            rf = json.load(rf)

            try:
                objs = rf['shapes']
            except:
                continue

            boxs = []
            for o in objs:
                try:
                    id = o['label']
                    if clsStaticFlag:
                        if id in clsStatic.keys():
                            clsStatic[id] += 1
                        else:
                            clsStatic[id] = 1
                    if savetxtPathFlag:
                        if id not in namell.keys():
                            continue
                        id = str(namell[id])

                        xmin = o['points'][0][0]
                        ymin = o['points'][0][1]
                        xmax = o['points'][1][0]
                        ymax = o['points'][1][1]

                        ymax = imgh if (ymax > imgh or (imgh - ymax < 5)) else ymax
                        ymin = 0 if ymin < 5 else ymin
                        xmax = imgw if (xmax > imgw or (imgw - xmax < 5)) else xmax
                        xmin = 0 if xmin < 5 else xmin
                        x = (xmin + xmax) / 2 / imgw
                        y = (ymin + ymax) / 2 / imgh
                        w = (xmax - xmin) / imgw
                        h = (ymax - ymin) / imgh
                        x = str(x) if len(str(x)) <= 7 else str(x)[:7]
                        y = str(y) if len(str(y)) <= 7 else str(y)[:7]
                        w = str(w) if len(str(w)) <= 7 else str(w)[:7]
                        h = str(h) if len(str(h)) <= 7 else str(h)[:7]
                        box = id + ' ' + x + ' ' + y + ' ' + w + ' ' + h + '\n'
                        boxs.append(box)
                except Exception as e:
                    print('jie Xi Json Error: ', e)
                    continue
            if boxs == [] and savetxtPathFlag == True:
                continue

        if savetxtPathFlag:
            boxs[-1] = boxs[-1].replace('\n', '')
            txt_path_name = os.path.join(savetxtPath, f[:-4] + '.txt')
            with open(txt_path_name, 'w', encoding='utf-8') as wf:
                wf.writelines(boxs)
    if clsStaticFlag:
        print('clsStatic: ', clsStatic)

        # break