Tools/GetTrainValTestCSV.py

import os
import glob
import random

import pandas as pd
import cv2
import tqdm
import numpy as np


class GetTrainTestCSV:
    def __init__(self, dataset_path_list, csv_name, img_format_list, negative_keep_rate=0.1):
        self.data_path_list = dataset_path_list
        self.img_format_list = img_format_list
        self.negative_keep_rate = negative_keep_rate
        self.save_path_csv = r'generate_dep_info'
        os.makedirs(self.save_path_csv, exist_ok=True)
        self.csv_name = csv_name

    def get_csv(self, pattern):
        def get_data_infos(img_path, img_format):
            data_info = {'img': [], 'label': []}
            img_file_list = glob.glob(img_path + '/*%s' % img_format)
            assert len(img_file_list), 'No data in DATASET_PATH!'
            for img_file in tqdm.tqdm(img_file_list):
                label_file = img_file.replace(img_format, 'png').replace('imgs', 'labels')
                if not os.path.exists(label_file):
                    label_file = 'None'
                # if os.path.getsize(label_file) == 0:
                #     if np.random.random() < self.negative_keep_rate:
                #         data_info['img'].append(img_file)
                #         data_info['label'].append(label_file)
                #     continue
                if pattern == 'test':
                    label_file = 'None'
                data_info['img'].append(img_file)
                data_info['label'].append(label_file)

            return data_info

        data_information = {'img': [], 'label': []}
        for idx, data_dir in enumerate(self.data_path_list):
            if len(self.data_path_list) == len(self.img_format_list):
                img_format = self.img_format_list[idx]
            else:
                img_format = self.img_format_list[0]
            assert os.path.exists(data_dir), 'No dir: ' + data_dir
            img_path_list = glob.glob(data_dir+'/*{0}'.format(img_format))
            # img folder
            if len(img_path_list) == 0:
                img_path_list = glob.glob(data_dir+'/*')
                for img_path in img_path_list:
                    if os.path.isdir(img_path):
                        data_info = get_data_infos(img_path, img_format)
                        data_information['img'].extend(data_info['img'])
                        data_information['label'].extend(data_info['label'])

            else:
                data_info = get_data_infos(data_dir, img_format)
                data_information['img'].extend(data_info['img'])
                data_information['label'].extend(data_info['label'])

        data_annotation = pd.DataFrame(data_information)
        writer_name = self.save_path_csv + '/' + self.csv_name
        data_annotation.to_csv(writer_name, index_label=False)
        print(os.path.basename(writer_name) + ' file saves successfully!')

    def generate_val_data_from_train_data(self, frac=0.1):
        if os.path.exists(self.save_path_csv + '/' + self.csv_name):
            data = pd.read_csv(self.save_path_csv + '/' + self.csv_name)
        else:
            raise Exception('no train data')

        val_data = data.sample(frac=frac, replace=False)
        train_data = data.drop(val_data.index)
        val_data = val_data.reset_index(drop=True)
        train_data = train_data.reset_index(drop=True)
        writer_name = self.save_path_csv + '/' + self.csv_name
        train_data.to_csv(writer_name, index_label=False)
        writer_name = self.save_path_csv + '/' + self.csv_name.replace('train', 'val')
        val_data.to_csv(writer_name, index_label=False)

    def _get_file(self, in_path_list):
        file_list = []
        for file in in_path_list:
            if os.path.isdir(os.path.abspath(file)):
                files = glob.glob(file + '/*')
                file_list.extend(self._get_file(files))
            else:
                file_list += [file]
        return file_list

    def get_csv_file(self, phase):
        phases = ['seg', 'flow', 'od']
        assert phase in phases, f'{phase} should in {phases}!'

        file_list = self._get_file(self.data_path_list)
        file_list = [x for x in file_list if x.split('.')[-1] in self.img_format_list]
        assert len(file_list), 'No data in  data_path_list!'
        random.shuffle(file_list)
        data_information = {}
        if phase == 'seg':
            data_information['img'] = file_list
            data_information['label'] = [x.replace('img', 'label') for x in file_list]
        elif phase == 'flow':
            data_information['img1'] = file_list[:-1]
            data_information['img2'] = file_list[1:]
        elif phase == 'od':
            data_information['img'] = file_list
            data_information['label'] = [x.replace('tiff', 'txt').replace('jpg', 'txt').replace('png', 'txt') for x in file_list]

        data_annotation = pd.DataFrame(data_information)
        writer_name = self.save_path_csv + '/' + self.csv_name
        data_annotation.to_csv(writer_name, index_label=False)
        print(os.path.basename(writer_name) + ' file saves successfully!')


if __name__ == '__main__':
    data_path_list = [
        'D:/Code/ProjectOnGithub/STT/Data/val_samples/img'
                      ]
    csv_name = 'val_data.csv'
    img_format_list = ['png']

    getTrainTestCSV = GetTrainTestCSV(dataset_path_list=data_path_list, csv_name=csv_name, img_format_list=img_format_list)
    getTrainTestCSV.get_csv_file(phase='seg')