dev

app.py

@@ -142,15 +142,14 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         except:                                            
             tokenizer = AutoTokenizer.from_pretrained('google/t5-efficient-tiny-nh8')
             # Tokenize the dataset                    
-            first_half = dataset['train'].select(range(third_size))
-            dataset['train'] = first_half
+            first_third = dataset['train'].select(range(third_size))
+            dataset['train'] = first_third
             del dataset['test']
             del dataset['validation']
-            tokenized_second_half = dataset.map(tokenize_function, batched=True)
-            tokenized_half = tokenize_function(first_half.to_dict())
+            tokenized_first_third = dataset.map(tokenize_function, batched=True)
             
-            tokenized_half.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')                           
-        
+            tokenized_first_third.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')                           
+            print('DONE')
             return 'RUN AGAIN TO LOAD REST OF DATA'
 
         # Fine-tune the model