dev

app.py

@@ -126,6 +126,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             if os.access(f'/data/{hub_id.strip()}_test_dataset', os.R_OK):
                 train_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset3')
                 saved_test_dataset = load_from_disk(f'/data/{hub_id.strip()}_validation_dataset')
+                dataset = load_dataset(dataset_name.strip())
                 print("FOUND TEST")                    
                 # Create Trainer
                 data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)
@@ -133,7 +134,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                     model=model,
                     args=training_args,
                     train_dataset=train_dataset,
-                    eval_dataset=saved_test_dataset,
+                    eval_dataset=dataset['validation'],
                     compute_metrics=compute_metrics,
                     data_collator=data_collator,
                     #processing_class=tokenizer,