Hack90 commited on
Commit
c692723
·
verified ·
1 Parent(s): 78adb91

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +7 -2
tokenizer.json CHANGED
@@ -51,7 +51,7 @@
51
  "model": {
52
  "type": "BPE",
53
  "dropout": null,
54
- "unk_token": null,
55
  "continuing_subword_prefix": null,
56
  "end_of_word_suffix": null,
57
  "fuse_unk": false,
@@ -59,11 +59,16 @@
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
 
62
  "a": 2,
63
  "c": 3,
64
  "g": 4,
65
  "t": 5,
66
- "n": 6
 
 
 
 
67
  },
68
  "merges": []
69
  }
 
51
  "model": {
52
  "type": "BPE",
53
  "dropout": null,
54
+ "unk_token": true,
55
  "continuing_subword_prefix": null,
56
  "end_of_word_suffix": null,
57
  "fuse_unk": false,
 
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
62
+ "<|padding|>": 8,
63
  "a": 2,
64
  "c": 3,
65
  "g": 4,
66
  "t": 5,
67
+ "n": 6,
68
+ "y":7,
69
+ "w":9,
70
+ "k":10,
71
+ "r":11
72
  },
73
  "merges": []
74
  }