dev

app.py

@@ -111,7 +111,6 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         #max_length = 512
         # Load the dataset
         dataset = load_dataset(dataset_name.strip())
-        print(dataset.keys())
         train_size = len(dataset['train'])
         third_size = train_size // 3
         max_length = model.get_input_embeddings().weight.shape[0]
@@ -139,7 +138,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 second_third = dataset['train'].select(range(third_size, third_size*2))
                 dataset['train'] = second_third
                 del dataset['test']
-                tokenized_sh_fq_dataset = tokenize_function(dataset, batched=True)
+                tokenized_sh_fq_dataset = dataset.map(tokenize_function, batched=True)
                 dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_sh_fq_dataset['train']])
                 dataset.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
                 return