monsoon-nlp
/

dna-blockdiff

Model card Files Files and versions Community

monsoon-nlp commited on Mar 24

Commit

8158986

·

1 Parent(s): 0a77499

eos

Files changed (1) hide show

tokenizer_config.json +1 -1

tokenizer_config.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "clean_up_tokenization_spaces": true,
-  "eos_token": null,
   "model_max_length": 1024,
   "tokenizer_class": "EsmTokenizer"
 }

 {
   "clean_up_tokenization_spaces": true,
+  "eos_token": "<eos>",
   "model_max_length": 1024,
   "tokenizer_class": "EsmTokenizer"
 }