Kevin Fink
commited on
Commit
·
727db6f
1
Parent(s):
2ebe0fb
init
Browse files
app.py
CHANGED
|
@@ -25,7 +25,7 @@ def fine_tune_model(model_name, dataset_name, hub_id, num_epochs, batch_size, lr
|
|
| 25 |
# Tokenize the dataset
|
| 26 |
def tokenize_function(examples):
|
| 27 |
max_length = 256
|
| 28 |
-
return tokenizer(examples['text'], padding='longest',
|
| 29 |
|
| 30 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
| 31 |
|
|
|
|
| 25 |
# Tokenize the dataset
|
| 26 |
def tokenize_function(examples):
|
| 27 |
max_length = 256
|
| 28 |
+
return tokenizer(examples['text'], padding='longest', truncation=True)
|
| 29 |
|
| 30 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
| 31 |
|