hawalurahman commited on
Commit
7547131
1 Parent(s): fd1bfab

Update tokenizer_config.json

Browse files

remove sep_token because it led to an error during inference

Files changed (1) hide show
  1. tokenizer_config.json +0 -1
tokenizer_config.json CHANGED
@@ -33,7 +33,6 @@
33
  "legacy": true,
34
  "model_max_length": 1000000000000000019884624838656,
35
  "pad_token": "<pad>",
36
- "sep_token": "[SEP]",
37
  "sp_model_kwargs": {},
38
  "tokenizer_class": "T5Tokenizer",
39
  "unk_token": "<unk>"
 
33
  "legacy": true,
34
  "model_max_length": 1000000000000000019884624838656,
35
  "pad_token": "<pad>",
 
36
  "sp_model_kwargs": {},
37
  "tokenizer_class": "T5Tokenizer",
38
  "unk_token": "<unk>"