File size: 1,662 Bytes
edcf5ee |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
"""
datacheck via dataloader Script ver： Feb 23th 21:00
loop the data and check if they are all cool
"""
import time
import torch
from torch import nn, optim
from torch.utils.data import DataLoader
from torchvision import models, datasets, transforms
import torch.nn.functional as func
from torchsummary import summary
import matplotlib.pyplot as plt
from torchvision import models
import ssl
import os
ssl._create_default_https_context = ssl._create_unverified_context
def data_loop(device, train_loader, check_minibatch=100):
model_time = time.time()
prev_time = model_time
index = 0
for data, label in train_loader:
data = data.to(device)
# at the checking time now
if index % check_minibatch == check_minibatch - 1:
check_index = index // check_minibatch + 1
now_time = time.time()
gap_time = now_time - prev_time
prev_time = now_time
print('index of ' + str(check_minibatch) + ' minibatch:', check_index, ' time used:', gap_time)
index += 1
print('all checked, time used:', time.time() - model_time)
if __name__ == '__main__':
data_path = r'/root/autodl-tmp/datasets/L'
edge_size = 224
transform_train = transforms.Compose([transforms.Resize([edge_size, edge_size]),transforms.ToTensor()])
train_data = datasets.ImageFolder(data_path, transform=transform_train)
train_loader = DataLoader(train_data, batch_size=500, shuffle=False, num_workers=32)
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data_loop(device, train_loader)
|