|
|
""" |
|
|
datacheck via dataloader Script ver： Feb 23th 21:00 |
|
|
loop the data and check if they are all cool |
|
|
""" |
|
|
import time |
|
|
import torch |
|
|
from torch import nn, optim |
|
|
from torch.utils.data import DataLoader |
|
|
from torchvision import models, datasets, transforms |
|
|
import torch.nn.functional as func |
|
|
from torchsummary import summary |
|
|
import matplotlib.pyplot as plt |
|
|
from torchvision import models |
|
|
import ssl |
|
|
import os |
|
|
|
|
|
ssl._create_default_https_context = ssl._create_unverified_context |
|
|
|
|
|
|
|
|
def data_loop(device, train_loader, check_minibatch=100): |
|
|
model_time = time.time() |
|
|
prev_time = model_time |
|
|
index = 0 |
|
|
|
|
|
for data, label in train_loader: |
|
|
data = data.to(device) |
|
|
|
|
|
|
|
|
if index % check_minibatch == check_minibatch - 1: |
|
|
check_index = index // check_minibatch + 1 |
|
|
now_time = time.time() |
|
|
gap_time = now_time - prev_time |
|
|
prev_time = now_time |
|
|
print('index of ' + str(check_minibatch) + ' minibatch:', check_index, ' time used:', gap_time) |
|
|
|
|
|
index += 1 |
|
|
|
|
|
print('all checked, time used:', time.time() - model_time) |
|
|
|
|
|
|
|
|
if __name__ == '__main__': |
|
|
data_path = r'/root/autodl-tmp/datasets/L' |
|
|
edge_size = 224 |
|
|
transform_train = transforms.Compose([transforms.Resize([edge_size, edge_size]),transforms.ToTensor()]) |
|
|
|
|
|
train_data = datasets.ImageFolder(data_path, transform=transform_train) |
|
|
train_loader = DataLoader(train_data, batch_size=500, shuffle=False, num_workers=32) |
|
|
|
|
|
os.environ['CUDA_VISIBLE_DEVICES'] = '0' |
|
|
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') |
|
|
|
|
|
data_loop(device, train_loader) |
|
|
|
