dev

app.py

@@ -207,9 +207,9 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         #dataset['train'] = dataset['train'].select(range(8000))
         dataset['train'] = dataset['train'].select(range(1000))
         dataset['validation'] = dataset['validation'].select(range(100))
-        train_set = dataset['train'].map(tokenize_function, batched=True, batch_size=50, remove_columns=column_names,)
-        valid_set = dataset['validation'].map(tokenize_function, batched=True, batch_size=50, remove_columns=column_names,)
-        
+        train_set = tokenize_function(dataset['train'])
+        valid_set = tokenize_function(dataset['validation'])
+        print(valid_set['text'])
 
         print('DONE')