dev

app.py

@@ -129,14 +129,13 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     compute_metrics=compute_metrics,
                 )
             
-            elif os.access(f'/data/{hub_id.strip()}_train_dataset3', os.R_OK):
-                if len(dataset['train']) == len(train_dataset['train']):                            
-                    dataset = load_dataset(dataset_name.strip())
-                    del dataset['train']
-                    del dataset['validation']
-                    test_set = dataset.map(tokenize_function, batched=True)
-                    test_set['test'].save_to_disk(f'/data/{hub_id.strip()}_test_dataset')
-                    return 'TRAINING DONE'
+            elif os.access(f'/data/{hub_id.strip()}_train_dataset3', os.R_OK):                        
+                dataset = load_dataset(dataset_name.strip())
+                del dataset['train']
+                del dataset['validation']
+                test_set = dataset.map(tokenize_function, batched=True)
+                test_set['test'].save_to_disk(f'/data/{hub_id.strip()}_test_dataset')
+                return 'TRAINING DONE'
                 
             elif os.access(f'/data/{hub_id.strip()}_validation_dataset', os.R_OK):                    
                 dataset = load_dataset(dataset_name.strip())