Record Open Image v4 dataset into VOC format

First use this code for initial conversion

Reference: Google Open Image v4 data set is downloaded according to the specified category and converted into VOC format in batches - Anleenoの屋(anleenoxu.top)

import math
import shutil

import cv2
from lxml.etree import Element, SubElement, tostring
from xml.dom.minidom import parseString
import os


# 对于每一个label.txt，制作VOC格式的xml文件
# param1:../OID/UnRestraint_Of_MarineCreature_In_OpenImagev4/Dataset/train/Seahorse/
# param2:imglabel.txt
# param3:annotation
def make_xml(txt_root_path, txt_name_path, save_xml_path):
    txt_path = txt_root_path + "Label/" + txt_name_path
    pic_path = txt_root_path
    pic = cv2.imread(pic_path + txt_name_path.replace(".txt", ".jpg"))
    shutil.copy(pic_path + txt_name_path.replace(".txt", ".jpg"), "VOC2007/JPEGImages")
    shape = pic.shape

    f = open(txt_path, 'r')
    data = []
    for line in f:
        data.append(line.strip())
    print(data)
    f.close()

    # lines = str(line).split(' ')
    # box_num = len(data)
    pic_name = txt_name_path.replace(".txt", ".jpg")

    node_root = Element('annotation')
    node_folder = SubElement(node_root, 'folder')
    node_folder.text = 'JPEGImages'
    node_filename = SubElement(node_root, 'filename')
    # 图片名字
    node_filename.text = pic_name
    # 图像尺寸
    node_size = SubElement(node_root, 'size')
    node_width = SubElement(node_size, 'width')
    node_width.text = str(shape[1])

    node_height = SubElement(node_size, 'height')
    node_height.text = str(shape[0])

    node_depth = SubElement(node_size, 'depth')
    node_depth.text = str(shape[2])
    # 第二层循环遍历有多少个框
    for dat in data:
        dat = dat.split(" ")
        print(len(dat))
        print(dat)
        # 分支把标签合并
        if len(dat) == 5:
            Label = dat[0]

            Xmin = math.floor(float(dat[1]))
            Ymin = math.floor(float(dat[2]))
            Xmax = math.ceil(float(dat[3]))
            Ymax = math.ceil(float(dat[4]))

            # Xmin = int(float(dat[1]))
            # Ymin = int(float(dat[2]))
            # Xmax = int(float(dat[3]))
            # Ymax = int(float(dat[4]))
            print(Label, Xmin, Ymin, Xmax, Ymax)
            # img = cv2.rectangle(img, (Xmin, Ymin), (Xmax, Ymax), (0, 255, 0), 1)

            cls_name = Label

            node_object = SubElement(node_root, 'object')
            node_name = SubElement(node_object, 'name')
            # 类别名字
            node_name.text = cls_name

            node_bndbox = SubElement(node_object, 'bndbox')
            node_xmin = SubElement(node_bndbox, 'xmin')
            node_xmin.text = str(Xmin)
            node_ymin = SubElement(node_bndbox, 'ymin')
            node_ymin.text = str(Ymin)
            node_xmax = SubElement(node_bndbox, 'xmax')
            node_xmax.text = str(Xmax)
            node_ymax = SubElement(node_bndbox, 'ymax')
            node_ymax.text = str(Ymax)

        else:
            Label = dat[0] + dat[1]

            Xmin = math.floor(float(dat[2]))
            Ymin = math.floor(float(dat[3]))
            Xmax = math.ceil(float(dat[4]))
            Ymax = math.ceil(float(dat[5]))

            # Xmin = int(float(dat[2]))
            # Ymin = int(float(dat[3]))
            # Xmax = int(float(dat[4]))
            # Ymax = int(float(dat[5]))
            print(Label, Xmin, Ymin, Xmax, Ymax)
            # img = cv2.rectangle(img, (Xmin, Ymin), (Xmax, Ymax), (0, 255, 0), 1)

            cls_name = Label

            node_object = SubElement(node_root, 'object')
            node_name = SubElement(node_object, 'name')
            # 类别名字
            node_name.text = cls_name
            node_difficult = SubElement(node_object, 'difficult')
            node_difficult.text = '0'

            node_bndbox = SubElement(node_object, 'bndbox')
            node_xmin = SubElement(node_bndbox, 'xmin')
            node_xmin.text = str(Xmin)
            node_ymin = SubElement(node_bndbox, 'ymin')
            node_ymin.text = str(Ymin)
            node_xmax = SubElement(node_bndbox, 'xmax')
            node_xmax.text = str(Xmax)
            node_ymax = SubElement(node_bndbox, 'ymax')
            node_ymax.text = str(Ymax)
    xml = tostring(node_root, pretty_print=True)
    dom = parseString(xml)
    # print xml 打印查看结果
    xml_name = pic_name.replace(".jpg", "")
    xml_name = os.path.join(save_xml_path, xml_name + '.xml')
    with open(xml_name, 'wb') as f:
        # f.write(dom.toprettyxml(indent='\t', encoding='utf-8'))
        f.write(xml)


def make_voc_dir():
    os.makedirs('VOC2007/Annotations')
    os.makedirs('VOC2007/ImageSets')
    os.makedirs('VOC2007/ImageSets/Main')
    os.makedirs('VOC2007/ImageSets/Layout')
    os.makedirs('VOC2007/ImageSets/Segmentation')
    os.makedirs('VOC2007/JPEGImages')
    os.makedirs('VOC2007/SegmentationClass')
    os.makedirs('VOC2007/SegmentationObject')


if __name__ == '__main__':
    dataset_folder = ["train", "validation", "test"]
    class_folder = ["15instruments"]
    root_path = "E:/study/OIDv4_ToolKit/OID/Dataset/"
    make_voc_dir()
    # 遍历train,val,test数据集
    for folder1 in dataset_folder:
        # ../OID/UnRestraint_Of_MarineCreature_In_OpenImagev4/Dataset/train/
        dataset_root_path = root_path + folder1 + "/"
        # 遍历具体类别
        for folder2 in class_folder:
            # ../OID/UnRestraint_Of_MarineCreature_In_OpenImagev4/Dataset/train/Seahorse/
            label_root_path = dataset_root_path + folder2 + "/"

            save_xml_path = 'VOC2007/Annotations'

            # ../OID/UnRestraint_Of_MarineCreature_In_OpenImagev4/Dataset/train/Seahorse/Label/
            files = os.listdir(label_root_path + "/Label/")
            print(files)
            # 遍历label和box的标签文件
            for file in files:
                # 对于每一个label.txt:
                # param1:../OID/UnRestraint_Of_MarineCreature_In_OpenImagev4/Dataset/train/Seahorse/
                # param2:imglabel.txt
                # param3:annotation
                make_xml(label_root_path, file, save_xml_path)

After the modification, related folders and Annotation files are generated, and the pictures are copied to JPEGImages

 The names of annotation and image are still named after the pictures at that time, but I saw that many articles have written naming requirements, preferably five digits and  six digits , so I changed the file name with the following code (note that it is 6 digits instead of 5 digits, I don’t know where I saw the 5 digits!)

import os
import glob

# folder_path = r'E:\study\OIDv4_ToolKit\OID\Dataset\VOC2007\JPEGImages'
folder_path = 'E:\study\OIDv4_ToolKit\OID\Dataset\VOC2007\Annotation'
num = 1

if __name__ == '__main__':

    for file in glob.glob(os.path.join(folder_path, "*.xml")):
        s = '%05d' % num  # 前面补零占位
        os.rename(os.path.join(folder_path, file), os.path.join(folder_path, str(s) + '.xml'))
        num += 1

But the xml file in the annotation still has picture name information, so the following code is used to modify the filename content in the xml file

Reference: Python modify xml file content_python modify xml content_pika insect's blog-CSDN blog

import xml.etree.ElementTree as ET
import os
 
 
# 批量修改整个文件夹所有的xml文件
def change_all_xml(xml_path):
    filelist = os.listdir(xml_path)
    print(filelist)
    # 打开xml文档
    for xmlfile in filelist:
        xmlpath = os.path.join(xml_path, xmlfile)
        doc = ET.parse(xmlpath)
        root = doc.getroot()
        sub1 = root.find('filename')  # 找到filename标签，
        sub1.text = xmlfile  # 修改标签内容
 
        doc.write(xmlpath)  # 保存修改
 
 
# 修改某个特定的xml文件
def change_one_xml(xml_path):          # 输入的是这个xml文件的全路径
    # 打开xml文档
    doc = ET.parse(xml_path)
    root = doc.getroot()
    sub1 = root.find('filename')       # 找到filename标签，
    sub1.text = '07_205.jpg'           # 修改标签内容
    doc.write(xml_path)  # 保存修改
    print('----------done--------')
 
 
change_all_xml(r'E:\study\OIDv4_ToolKit\OID\Dataset\VOC2007\Annotations')     # xml文件总路径
# xml_path = r'Z:\pycharm_projects\ssd\VOC2007\Annotations\07_205.xml'
# change_one_xml(xml_path)

Next create and add the files under the main folder

import os
import random
 
trainval_percent = 0.9
train_percent = 0.8
xmlfilepath = 'E:/study/OIDv4_ToolKit/OID/Dataset/VOC2007/Annotations'
txtsavepath = 'E:/study/OIDv4_ToolKit/OID/Dataset/VOC2007/ImageSets/Main'
total_xml = os.listdir(xmlfilepath)
 
num=len(total_xml)
list=range(num)
tv=int(num*trainval_percent)
tr=int(tv*train_percent)
trainval= random.sample(list,tv)
train=random.sample(trainval,tr)
 
ftrainval = open(txtsavepath+'/trainval.txt', 'w')
ftest = open(txtsavepath+'/test.txt', 'w')
ftrain = open(txtsavepath+'/train.txt', 'w')
fval = open(txtsavepath+'/val.txt', 'w')
 
for i  in list:
    name=total_xml[i][:-4]+'\n'
    if i in trainval:
        ftrainval.write(name)
        if i in train:
            ftrain.write(name)
        else:
            fval.write(name)
    else:
        ftest.write(name)
 
ftrainval.close()
ftrain.close()
fval.close()
ftest .close()

It is found that the object needs to be in lowercase, but the category of Open Image is capitalized and needs to be modified, otherwise KeyError may occur, use the following code to modify (remember to synchronously modify the category name in pacal_voc.py to lowercase)

Reference: Python modifies the category name in the xml file_Soochow_NJU_Smile's Blog-CSDN Blog

#修改xml文件中的目标的名字
import os, sys
import glob
from xml.etree import ElementTree as ET

# 批量读取Annotations下的xml文件
# per=ET.parse(r'C:\Users\xxxxx\Desktop\Annotations\000003.xml')
xml_dir = r'E:\study\OIDv4_ToolKit\OID\Dataset\VOC2007\Annotations'
xml_list = glob.glob(xml_dir + '/*.xml')
for xml in xml_list:
    print(xml)
    per = ET.parse(xml)
    p = per.findall('/object')

    for oneper in p:  # 找出person节点
        child = oneper.getchildren()[0]  # 找出person节点的子节点
        child.text = child.text.lower()
        # if child.text == 'yanwu':   #需要修改的名字
        #     child.text = 'smoke'    #修改成什么名字

    per.write(xml)
    print(child.tag, ':', child.text)