dev

app.py

@@ -117,32 +117,36 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         third_size = train_size // 3
         max_length = model.get_input_embeddings().weight.shape[0]
         try:
-            saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')            
-            if 'test' in saved_dataset.keys():
-                print("FOUND TEST")
-                # Create Trainer
-                trainer = Trainer(
-                    model=model,
-                    args=training_args,
-                    train_dataset=tokenized_train_dataset,
-                    eval_dataset=tokenized_test_dataset,
-                    compute_metrics=compute_metrics,
-                )                   
-            elif 'validation' in saved_dataset.keys():
-                print("FOUND VALIDATION")
-                third_third = dataset['train'].select(range(third_size*2, train_size))
-                dataset['train'] = third_third
-                tokenized_second_half = dataset.map(tokenize_function, batched=True)
-                dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_second_half['train']])
-                tokenized_train_dataset = dataset['train']
-                tokenized_test_dataset = dataset['test']
+            saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')                                       
+            if 'validation' in saved_dataset.keys():
+                if 'test' in saved_dataset.keys():
+                    print("FOUND TEST")
+                    dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset3')
+                    # Create Trainer
+                    trainer = Trainer(
+                        model=model,
+                        args=training_args,
+                        train_dataset=tokenized_train_dataset,
+                        eval_dataset=tokenized_test_dataset,
+                        compute_metrics=compute_metrics,
+                    )
+                else:
+                    print("FOUND VALIDATION")
+                    saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset2')     
+                    third_third = dataset['train'].select(range(third_size*2, train_size))
+                    dataset['train'] = third_third
+                    tokenized_second_half = dataset.map(tokenize_function, batched=True)
+                    dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_second_half['train']])
+                    tokenized_train_dataset = dataset['train']
+                    tokenized_test_dataset = dataset['test']
+                    dataset.save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')
             else:
                 second_third = dataset['train'].select(range(third_size, third_size*2))
                 dataset['train'] = second_third
                 del dataset['test']
                 tokenized_sh_fq_dataset = dataset.map(tokenize_function, batched=True)
                 dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_sh_fq_dataset['train']])
-                dataset.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
+                dataset.save_to_disk(f'/data/{hub_id.strip()}_train_dataset2')
                 return
                 
         except: