dev

app.py

@@ -117,7 +117,8 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         try:
             tokenized_first_half = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
             second_half = dataset['train'].select(range(half_size, train_size))
-            tokenized_second_half = tokenize_function(second_half.to_dict())
+            dataset['train'] = second_half
+            tokenized_second_half = dataset.map(tokenize_function, batched=True)
             tokenized_train_dataset = concatenate_datasets([tokenized_first_half, tokenized_second_half])
             tokenized_test_dataset = tokenize_function(dataset['test'])
             
@@ -133,6 +134,10 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
             tokenizer = AutoTokenizer.from_pretrained('google/t5-efficient-tiny-nh8')
             # Tokenize the dataset                    
             first_half = dataset['train'].select(range(half_size))
+            dataset['train'] = first_half
+            del dataset['test']
+            del dataset['validation']
+            tokenized_second_half = dataset.map(tokenize_function, batched=True)
             tokenized_half = tokenize_function(first_half.to_dict())
             
             tokenized_half.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')                           
@@ -211,5 +216,4 @@ try:
     # Launch the interface
     iface.launch()    
 except Exception as e:
-    print(f"An error occurred: {str(e)}, TB: {traceback.format_exc()}")
-
+    print(f"An error occurred: {str(e)}, TB: {traceback.format_exc()}")