Hack90 commited on
Commit
14efbe7
·
verified ·
1 Parent(s): b7b59a4

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -0
tokenizer.json CHANGED
@@ -73,6 +73,7 @@
73
  "vocab": {
74
  "<|endoftext|>": 0,
75
  "<|padding|>": 1,
 
76
  "a": 28,
77
  "c": 29,
78
  "g": 30,
 
73
  "vocab": {
74
  "<|endoftext|>": 0,
75
  "<|padding|>": 1,
76
+ "<|unknown|>" : 2
77
  "a": 28,
78
  "c": 29,
79
  "g": 30,