Hack90 commited on
Commit
da0fa7f
·
verified ·
1 Parent(s): 5fe7d19

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -6
tokenizer.json CHANGED
@@ -51,7 +51,7 @@
51
  "model": {
52
  "type": "BPE",
53
  "dropout": null,
54
- "unk_token": true,
55
  "continuing_subword_prefix": null,
56
  "end_of_word_suffix": null,
57
  "fuse_unk": false,
@@ -59,16 +59,12 @@
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
62
- "<|unk|>": 8,
63
  "a": 2,
64
  "c": 3,
65
  "g": 4,
66
  "t": 5,
67
  "n": 6,
68
- "y":7,
69
- "w":9,
70
- "k":10,
71
- "r":11
72
  },
73
  "merges": []
74
  }
 
51
  "model": {
52
  "type": "BPE",
53
  "dropout": null,
54
+ "unk_token": null,
55
  "continuing_subword_prefix": null,
56
  "end_of_word_suffix": null,
57
  "fuse_unk": false,
 
59
  "vocab": {
60
  "<|endoftext|>": 0,
61
  "<|padding|>": 1,
 
62
  "a": 2,
63
  "c": 3,
64
  "g": 4,
65
  "t": 5,
66
  "n": 6,
67
+ "y":7
 
 
 
68
  },
69
  "merges": []
70
  }