dev

app.py

@@ -112,22 +112,22 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         # Load the dataset
         dataset = load_dataset(dataset_name.strip())          
         train_size = len(dataset['train'])
-        half_size = train_size // 2  
+        third_size = train_size // 3
         max_length = model.get_input_embeddings().weight.shape[0]
         try:
-            tokenized_first_half = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
-            if 'test' in tokenized_first_half.keys():                
-                second_half_second_quarter = dataset['train'].select(range(half_size+half_size//2, train_size))
-                dataset['train'] = second_half_second_quarter
+            saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
+            if 'test' in saved_dataset.keys():                
+                third_third = dataset['train'].select(range(third_size*2, train_size))
+                dataset['train'] = third_third
                 tokenized_second_half = dataset.map(tokenize_function, batched=True)
-                dataset['train'] = concatenate_datasets([tokenized_first_half['train'], tokenized_second_half['train']])
+                dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_second_half['train']])
                 tokenized_train_dataset = dataset['train']
                 tokenized_test_dataset = dataset['test']
             else:
-                second_half_first_quarter = dataset['train'].select(range(half_size, half_size+half_size//2))
-                dataset['train'] = second_half_first_quarter                
+                second_third = dataset['train'].select(range(third_size, third_size*2))
+                dataset['train'] = second_third                
                 tokenized_sh_fq_dataset = tokenize_function(dataset, batched=True)
-                dataset['train'] = concatenate_datasets([tokenized_first_half['train'], tokenized_sh_fq_dataset['train']])
+                dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_sh_fq_dataset['train']])
                 tokenized_half.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
                 return
             
@@ -142,7 +142,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         except:                                            
             tokenizer = AutoTokenizer.from_pretrained('google/t5-efficient-tiny-nh8')
             # Tokenize the dataset                    
-            first_half = dataset['train'].select(range(half_size))
+            first_half = dataset['train'].select(range(third_size))
             dataset['train'] = first_half
             del dataset['test']
             del dataset['validation']