dev

app.py

@@ -207,7 +207,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         #dataset['train'] = dataset['train'].select(range(8000))
         dataset['train'] = dataset['train'].select(range(1000))
         dataset['validation'] = dataset['validation'].select(range(100))
-        tokenized_first_third = dataset.map(tokenize_function, batched=True, batch_size=50, remove_columns=column_names,)
+        train_set = dataset['train'].map(tokenize_function, batched=True, batch_size=50, remove_columns=column_names,)
+        valid_set = dataset['validation'].map(tokenize_function, batched=True, batch_size=50, remove_columns=column_names,)
         
 
         print('DONE')
@@ -217,8 +218,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         trainer = Trainer(
             model=model,
             args=training_args,
-            train_dataset=dataset['train'],
-            eval_dataset=dataset['validation'],
+            train_dataset=train_set, 
+            eval_dataset=valid_set,
             compute_metrics=compute_metrics,
             data_collator=data_collator,
             #processing_class=tokenizer,