Hack90 commited on
Commit
e7e81f8
·
verified ·
1 Parent(s): 210ee5b

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +3 -2
tokenizer.json CHANGED
@@ -71,13 +71,14 @@
71
  "fuse_unk": false,
72
  "byte_fallback": false,
73
  "vocab": {
74
- "<|endoftext|>": 0,
75
  "<|padding|>": 1,
76
  "<|unknown|>" : 2,
77
  "a": 28,
78
  "c": 29,
79
  "g": 30,
80
  "t": 31,
81
- "n": 32
 
82
  }, "merges":[] }}
83
 
 
71
  "fuse_unk": false,
72
  "byte_fallback": false,
73
  "vocab": {
74
+ "<|endoftext|>": 1,
75
  "<|padding|>": 1,
76
  "<|unknown|>" : 2,
77
  "a": 28,
78
  "c": 29,
79
  "g": 30,
80
  "t": 31,
81
+ "n": 32,
82
+ "<|beginoftext|>" : 0
83
  }, "merges":[] }}
84