Spaces:

shorecode
/

gradio-3

Sleeping

Kevin Fink commited on Dec 8, 2024

Commit

c932e54

1 Parent(s): 1e30251

dev

Files changed (1) hide show

app.py CHANGED Viewed

@@ -152,7 +152,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 print(dataset.keys())
                 #tokenized_second_half = tokenize_function(third_third)
                 tokenized_second_half = dataset.map(tokenize_function, batched=True)
-                dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_second_half['train']])
                 dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')
                 return 'THIRD THIRD LOADED'

                 print(dataset.keys())
                 #tokenized_second_half = tokenize_function(third_third)
                 tokenized_second_half = dataset.map(tokenize_function, batched=True)
+                dataset['train'] = concatenate_datasets([saved_dataset, tokenized_second_half['train']])
                 dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')
                 return 'THIRD THIRD LOADED'