Kevin Fink
commited on
Commit
·
727db6f
1
Parent(s):
2ebe0fb
init
Browse files
app.py
CHANGED
@@ -25,7 +25,7 @@ def fine_tune_model(model_name, dataset_name, hub_id, num_epochs, batch_size, lr
|
|
25 |
# Tokenize the dataset
|
26 |
def tokenize_function(examples):
|
27 |
max_length = 256
|
28 |
-
return tokenizer(examples['text'], padding='longest',
|
29 |
|
30 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
31 |
|
|
|
25 |
# Tokenize the dataset
|
26 |
def tokenize_function(examples):
|
27 |
max_length = 256
|
28 |
+
return tokenizer(examples['text'], padding='longest', truncation=True)
|
29 |
|
30 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
31 |
|