dev

app.py

@@ -116,7 +116,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         print(f"TESTING: {os.access(f'/data/{hub_id.strip()}_train_dataset', os.R_OK)}")
         try:
             saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
-            if os.path.isfile(f'/data/{hub_id.strip()}_test_dataset'):
+            if os.access(f'/data/{hub_id.strip()}_test_dataset', os.R_OK):
                 train_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset3')
                 saved_test_dataset = load_from_disk(f'/data/{hub_id.strip()}_test_dataset')
                 print("FOUND TEST")                    
@@ -129,7 +129,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     compute_metrics=compute_metrics,
                 )
             
-            elif os.path.isfile(f'/data/{hub_id.strip()}_train_dataset3'):
+            elif os.access(f'/data/{hub_id.strip()}_train_dataset3', os.R_OK):
                 if len(dataset['train']) == len(train_dataset['train']):                            
                     dataset = load_dataset(dataset_name.strip())
                     del dataset['train']
@@ -138,7 +138,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     test_set['test'].save_to_disk(f'/data/{hub_id.strip()}_test_dataset')
                     return 'TRAINING DONE'
                 
-            elif os.path.isfile(f'/data/{hub_id.strip()}_validation_dataset'):                    
+            elif os.access(f'/data/{hub_id.strip()}_validation_dataset', os.R_OK):                    
                 dataset = load_dataset(dataset_name.strip())
                 train_size = len(dataset['train'])
                 third_size = train_size // 3
@@ -157,7 +157,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 return 'THIRD THIRD LOADED'
             
                                            
-            if os.path.isfile(f'/data/{hub_id.strip()}_train_dataset' and not os.path.isfile(f'/data/{hub_id.strip()}_train_dataset3')):
+            if os.access(f'/data/{hub_id.strip()}_train_dataset', os.R_OK) and not os.access(f'/data/{hub_id.strip()}_train_dataset3', os.R_OK):
                 dataset = load_dataset(dataset_name.strip())
                 train_size = len(dataset['train'])
                 third_size = train_size // 3