Zual commited on
Commit
d658e9b
1 Parent(s): dcea1b2

Upload tokenizer_data.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_data.json +203 -0
tokenizer_data.json ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "vocab": {
3
+ "<PAD>": 0,
4
+ "<UNK>": 1,
5
+ "a1": 2,
6
+ "a2": 3,
7
+ "a3": 4,
8
+ "a4": 5,
9
+ "a5": 6,
10
+ "a6": 7,
11
+ "a7": 8,
12
+ "a8": 9,
13
+ "b1": 10,
14
+ "b2": 11,
15
+ "b3": 12,
16
+ "b4": 13,
17
+ "b5": 14,
18
+ "b6": 15,
19
+ "b7": 16,
20
+ "b8": 17,
21
+ "c1": 18,
22
+ "c2": 19,
23
+ "c3": 20,
24
+ "c4": 21,
25
+ "c5": 22,
26
+ "c6": 23,
27
+ "c7": 24,
28
+ "c8": 25,
29
+ "d1": 26,
30
+ "d2": 27,
31
+ "d3": 28,
32
+ "d4": 29,
33
+ "d5": 30,
34
+ "d6": 31,
35
+ "d7": 32,
36
+ "d8": 33,
37
+ "e1": 34,
38
+ "e2": 35,
39
+ "e3": 36,
40
+ "e4": 37,
41
+ "e5": 38,
42
+ "e6": 39,
43
+ "e7": 40,
44
+ "e8": 41,
45
+ "f1": 42,
46
+ "f2": 43,
47
+ "f3": 44,
48
+ "f4": 45,
49
+ "f5": 46,
50
+ "f6": 47,
51
+ "f7": 48,
52
+ "f8": 49,
53
+ "g1": 50,
54
+ "g2": 51,
55
+ "g3": 52,
56
+ "g4": 53,
57
+ "g5": 54,
58
+ "g6": 55,
59
+ "g7": 56,
60
+ "g8": 57,
61
+ "h1": 58,
62
+ "h2": 59,
63
+ "h3": 60,
64
+ "h4": 61,
65
+ "h5": 62,
66
+ "h6": 63,
67
+ "h7": 64,
68
+ "h8": 65,
69
+ "1": 66,
70
+ ".": 67,
71
+ " ": 68,
72
+ "2": 69,
73
+ "N": 70,
74
+ "3": 71,
75
+ "c": 72,
76
+ "x": 73,
77
+ "4": 74,
78
+ "5": 75,
79
+ "B": 76,
80
+ "6": 77,
81
+ "7": 78,
82
+ "8": 79,
83
+ "b": 80,
84
+ "9": 81,
85
+ "0": 82,
86
+ "e": 83,
87
+ "O": 84,
88
+ "-": 85,
89
+ "Q": 86,
90
+ "R": 87,
91
+ "f": 88,
92
+ "a": 89,
93
+ "d": 90,
94
+ "K": 91,
95
+ "g": 92,
96
+ "+": 93,
97
+ "#": 94,
98
+ "h": 95,
99
+ "=": 96,
100
+ "/": 97
101
+ },
102
+ "ids_to_tokens": {
103
+ "0": "<PAD>",
104
+ "1": "<UNK>",
105
+ "2": "a1",
106
+ "3": "a2",
107
+ "4": "a3",
108
+ "5": "a4",
109
+ "6": "a5",
110
+ "7": "a6",
111
+ "8": "a7",
112
+ "9": "a8",
113
+ "10": "b1",
114
+ "11": "b2",
115
+ "12": "b3",
116
+ "13": "b4",
117
+ "14": "b5",
118
+ "15": "b6",
119
+ "16": "b7",
120
+ "17": "b8",
121
+ "18": "c1",
122
+ "19": "c2",
123
+ "20": "c3",
124
+ "21": "c4",
125
+ "22": "c5",
126
+ "23": "c6",
127
+ "24": "c7",
128
+ "25": "c8",
129
+ "26": "d1",
130
+ "27": "d2",
131
+ "28": "d3",
132
+ "29": "d4",
133
+ "30": "d5",
134
+ "31": "d6",
135
+ "32": "d7",
136
+ "33": "d8",
137
+ "34": "e1",
138
+ "35": "e2",
139
+ "36": "e3",
140
+ "37": "e4",
141
+ "38": "e5",
142
+ "39": "e6",
143
+ "40": "e7",
144
+ "41": "e8",
145
+ "42": "f1",
146
+ "43": "f2",
147
+ "44": "f3",
148
+ "45": "f4",
149
+ "46": "f5",
150
+ "47": "f6",
151
+ "48": "f7",
152
+ "49": "f8",
153
+ "50": "g1",
154
+ "51": "g2",
155
+ "52": "g3",
156
+ "53": "g4",
157
+ "54": "g5",
158
+ "55": "g6",
159
+ "56": "g7",
160
+ "57": "g8",
161
+ "58": "h1",
162
+ "59": "h2",
163
+ "60": "h3",
164
+ "61": "h4",
165
+ "62": "h5",
166
+ "63": "h6",
167
+ "64": "h7",
168
+ "65": "h8",
169
+ "66": "1",
170
+ "67": ".",
171
+ "68": " ",
172
+ "69": "2",
173
+ "70": "N",
174
+ "71": "3",
175
+ "72": "c",
176
+ "73": "x",
177
+ "74": "4",
178
+ "75": "5",
179
+ "76": "B",
180
+ "77": "6",
181
+ "78": "7",
182
+ "79": "8",
183
+ "80": "b",
184
+ "81": "9",
185
+ "82": "0",
186
+ "83": "e",
187
+ "84": "O",
188
+ "85": "-",
189
+ "86": "Q",
190
+ "87": "R",
191
+ "88": "f",
192
+ "89": "a",
193
+ "90": "d",
194
+ "91": "K",
195
+ "92": "g",
196
+ "93": "+",
197
+ "94": "#",
198
+ "95": "h",
199
+ "96": "=",
200
+ "97": "/"
201
+ },
202
+ "next_id": 98
203
+ }