Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
@@ -33,7 +33,7 @@ def main():
|
|
33 |
|
34 |
# Tokenize the datasets
|
35 |
def tokenize_function(examples):
|
36 |
-
return tokenizer(examples['
|
37 |
|
38 |
# 对训练集和验证集进行分词处理
|
39 |
train_dataset = train_dataset.map(tokenize_function, batched=True)
|
|
|
33 |
|
34 |
# Tokenize the datasets
|
35 |
def tokenize_function(examples):
|
36 |
+
return tokenizer(examples['question'], padding='max_length', truncation=True, max_length=128)
|
37 |
|
38 |
# 对训练集和验证集进行分词处理
|
39 |
train_dataset = train_dataset.map(tokenize_function, batched=True)
|