import os import numpy as np from PIL import Image from torch.utils.data import DataLoader, Dataset from torch.utils.data.sampler import SubsetRandomSampler import torch import torchvision.transforms as transforms from tqdm import tqdm from sklearn.model_selection import train_test_split from typing import List, Optional from glob import glob # dataset class for 3DCNN class Dataset_3DCNN(Dataset): "Characterizes a dataset for PyTorch" def __init__(self, path : str, folders : List[str], labels : List[float], frames : List[int], transform : Optional[transforms.Compose] = None): "Initialization" self.path = path self.labels = labels self.folders = folders self.transform = transform self.frames = frames def __len__(self): "Denotes the total number of samples" return len(self.folders) def read_images(self, path, selected_folder, use_transform): X = [] for i in self.frames: image = Image.open(os.path.join(path, selected_folder, 'frame_{:01d}.jpg'.format(i))).convert('L') if use_transform is not None: image = use_transform(image) else: image = transforms.ToTensor()(image) X.append(image.squeeze_(0)) X = torch.stack(X, dim=0) return X def __getitem__(self, index): "Generates one sample of data" # Select sample folder = self.folders[index] # Load data X = self.read_images(self.path, folder, self.transform).unsqueeze_(0) # (input) spatial images y = torch.LongTensor([self.labels[index]]) # (labels) LongTensor are for int64 instead of FloatTensor # print(X.shape) return X, y # train test splitting def create_datasets(path : str = r'D:\All_files\pys\AI_algos\Mikes_Work\viscosity-video-classification\code_digdiscovery\new_honey_164', # absolute path validation_split : float = 0.2, test_split : float = 0.2, batch_size : int = 32, transform : transforms.Compose = transforms.Compose([transforms.Resize([256, 342]), transforms.ToTensor(), transforms.Normalize(mean=[0.5], std=[0.5])]), random_seed : int = 112, shuffle : bool = True, selected_frames : List[int] = [0,10,20]): all_X_list = [filename for filename in os.listdir(path)] all_y_list = [int(filename) for filename in os.listdir(path)] # train, test split train_list, test_list, train_label, test_label = train_test_split(all_X_list, all_y_list, test_size=test_split, random_state=random_seed) train_set, test_set = Dataset_3DCNN(path, train_list, train_label, selected_frames, transform=transform), \ Dataset_3DCNN(path, test_list, test_label, selected_frames, transform=transform) print('length test set ', len(test_set)) # split into training and validation batches num_train = len(train_list) indices = list(range(num_train)) if shuffle : np.random.seed(random_seed) np.random.shuffle(indices) split = int(np.floor(validation_split * num_train)) train_idx, valid_idx = indices[split:], indices[:split] train_sampler = SubsetRandomSampler(train_idx) valid_sampler = SubsetRandomSampler(valid_idx) # loading train, validation and test data train_loader = DataLoader(train_set, batch_size=batch_size, sampler=train_sampler, num_workers=0) valid_loader = DataLoader(train_set, batch_size=batch_size, sampler=valid_sampler, num_workers=0) test_loader = DataLoader(test_set, batch_size=batch_size, num_workers=0) return train_loader, test_loader, valid_loader def fetch_data_single_folder(path : str = r'C:\Users\bdutta\work\pys\AI_algos\viscosity\new_honey_164\2350', frames : np.array = np.arange(2,62,2), use_transform : transforms.Compose =transforms.Compose([transforms.Resize([256, 342]), transforms.ToTensor(), transforms.Normalize(mean=[0.5], std=[0.5])]) ): X = [] for i in frames: image = Image.open(os.path.join(path, 'frame_{:01d}.jpg'.format(i))).convert('L') if use_transform is not None: image = use_transform(image) else: image = transforms.ToTensor()(image) X.append(image) X = torch.stack(X, dim=1).unsqueeze(0) return X