Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -33,7 +33,7 @@ def main():
|
|
| 33 |
|
| 34 |
# Tokenize the datasets
|
| 35 |
def tokenize_function(examples):
|
| 36 |
-
return tokenizer(examples['
|
| 37 |
|
| 38 |
# 对训练集和验证集进行分词处理
|
| 39 |
train_dataset = train_dataset.map(tokenize_function, batched=True)
|
|
|
|
| 33 |
|
| 34 |
# Tokenize the datasets
|
| 35 |
def tokenize_function(examples):
|
| 36 |
+
return tokenizer(examples['question'], padding='max_length', truncation=True, max_length=128)
|
| 37 |
|
| 38 |
# 对训练集和验证集进行分词处理
|
| 39 |
train_dataset = train_dataset.map(tokenize_function, batched=True)
|