dev

app.py

@@ -142,13 +142,15 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 except:
                     dataset = load_dataset(dataset_name.strip())
                     train_size = len(dataset['train'])
-                    third_size = train_size // 3                    
+                    third_size = train_size // 3
+                    del dataset['test']
+                    del dataset['validation']                    
                     print("FOUND VALIDATION")
                     saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset2')     
                     third_third = dataset['train'].select(range(third_size*2, train_size))
                     dataset['train'] = third_third
-                    del dataset['test']
-                    del dataset['validation']
+                    print(dataset)
+                    print(dataset.keys())
                     tokenized_second_half = dataset.map(tokenize_function, batched=True)
                     dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_second_half['train']])
                     dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')