NX2411 commited on
Commit
9c2fe18
β€’
1 Parent(s): d051ca7

add tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +110 -110
vocab.json CHANGED
@@ -1,114 +1,114 @@
1
  {
2
- "1": 47,
3
- "2": 11,
4
- "3": 109,
5
- "4": 48,
6
- "5": 82,
7
- "7": 64,
8
- "8": 71,
9
- "A": 23,
10
- "B": 10,
11
- "C": 28,
12
- "D": 73,
13
- "E": 72,
14
- "F": 36,
15
- "G": 22,
16
- "H": 14,
17
- "I": 87,
18
- "J": 25,
19
- "K": 84,
20
- "L": 92,
21
- "M": 80,
22
- "N": 62,
23
- "O": 5,
24
- "P": 76,
25
- "Q": 105,
26
- "R": 86,
27
- "S": 50,
28
- "T": 7,
29
- "U": 65,
30
- "V": 90,
31
- "W": 16,
32
- "X": 17,
33
- "Y": 96,
34
- "Z": 2,
35
  "[PAD]": 111,
36
  "[UNK]": 110,
37
- "a": 44,
38
- "b": 100,
39
- "c": 70,
40
- "d": 37,
41
- "e": 68,
42
- "f": 1,
43
- "g": 13,
44
- "h": 52,
45
- "i": 18,
46
- "j": 27,
47
- "k": 39,
48
- "l": 75,
49
- "m": 20,
50
- "n": 91,
51
- "o": 32,
52
- "p": 35,
53
- "q": 40,
54
- "r": 97,
55
- "s": 79,
56
- "t": 30,
57
- "u": 106,
58
- "v": 77,
59
- "w": 93,
60
- "x": 3,
61
- "y": 51,
62
- "z": 60,
63
- "|": 46,
64
- "γ„±": 83,
65
- "γ„²": 98,
66
- "γ„³": 21,
67
- "γ„΄": 81,
68
- "γ„΅": 41,
69
- "γ„Ά": 55,
70
- "γ„·": 104,
71
- "γ„Έ": 34,
72
- "γ„Ή": 56,
73
- "γ„Ί": 59,
74
- "γ„»": 42,
75
- "γ„Ό": 94,
76
- "γ„Ύ": 103,
77
- "γ„Ώ": 99,
78
- "γ…€": 33,
79
- "ㅁ": 66,
80
- "γ…‚": 0,
81
- "γ…ƒ": 8,
82
- "γ…„": 31,
83
- "γ……": 19,
84
- "γ…†": 54,
85
- "γ…‡": 67,
86
- "γ…ˆ": 53,
87
- "γ…‰": 38,
88
- "γ…Š": 102,
89
- "γ…‹": 9,
90
- "γ…Œ": 101,
91
- "ㅍ": 12,
92
- "γ…Ž": 61,
93
- "ㅏ": 74,
94
- "ㅐ": 58,
95
- "γ…‘": 43,
96
- "γ…’": 108,
97
- "γ…“": 69,
98
- "γ…”": 63,
99
- "γ…•": 24,
100
- "γ…–": 78,
101
- "γ…—": 95,
102
- "γ…˜": 29,
103
- "γ…™": 57,
104
- "γ…š": 107,
105
- "γ…›": 4,
106
- "γ…œ": 88,
107
- "ㅝ": 89,
108
- "γ…ž": 26,
109
- "γ…Ÿ": 6,
110
- "γ… ": 85,
111
- "γ…‘": 45,
112
- "γ…’": 15,
113
- "γ…£": 49
114
  }
 
1
  {
2
+ "1": 82,
3
+ "2": 94,
4
+ "3": 12,
5
+ "4": 97,
6
+ "5": 81,
7
+ "7": 48,
8
+ "8": 68,
9
+ "A": 77,
10
+ "B": 79,
11
+ "C": 98,
12
+ "D": 100,
13
+ "E": 62,
14
+ "F": 26,
15
+ "G": 16,
16
+ "H": 23,
17
+ "I": 58,
18
+ "J": 91,
19
+ "K": 20,
20
+ "L": 88,
21
+ "M": 102,
22
+ "N": 86,
23
+ "O": 37,
24
+ "P": 42,
25
+ "Q": 80,
26
+ "R": 15,
27
+ "S": 70,
28
+ "T": 17,
29
+ "U": 11,
30
+ "V": 108,
31
+ "W": 29,
32
+ "X": 107,
33
+ "Y": 14,
34
+ "Z": 9,
35
  "[PAD]": 111,
36
  "[UNK]": 110,
37
+ "a": 78,
38
+ "b": 2,
39
+ "c": 67,
40
+ "d": 59,
41
+ "e": 69,
42
+ "f": 21,
43
+ "g": 53,
44
+ "h": 33,
45
+ "i": 89,
46
+ "j": 47,
47
+ "k": 71,
48
+ "l": 57,
49
+ "m": 13,
50
+ "n": 56,
51
+ "o": 34,
52
+ "p": 61,
53
+ "q": 19,
54
+ "r": 36,
55
+ "s": 27,
56
+ "t": 64,
57
+ "u": 93,
58
+ "v": 39,
59
+ "w": 25,
60
+ "x": 32,
61
+ "y": 38,
62
+ "z": 40,
63
+ "|": 24,
64
+ "γ„±": 43,
65
+ "γ„²": 46,
66
+ "γ„³": 22,
67
+ "γ„΄": 103,
68
+ "γ„΅": 109,
69
+ "γ„Ά": 74,
70
+ "γ„·": 41,
71
+ "γ„Έ": 76,
72
+ "γ„Ή": 83,
73
+ "γ„Ί": 66,
74
+ "γ„»": 51,
75
+ "γ„Ό": 4,
76
+ "γ„Ύ": 105,
77
+ "γ„Ώ": 31,
78
+ "γ…€": 87,
79
+ "ㅁ": 52,
80
+ "γ…‚": 55,
81
+ "γ…ƒ": 35,
82
+ "γ…„": 85,
83
+ "γ……": 92,
84
+ "γ…†": 50,
85
+ "γ…‡": 7,
86
+ "γ…ˆ": 90,
87
+ "γ…‰": 84,
88
+ "γ…Š": 8,
89
+ "γ…‹": 72,
90
+ "γ…Œ": 96,
91
+ "ㅍ": 60,
92
+ "γ…Ž": 44,
93
+ "ㅏ": 95,
94
+ "ㅐ": 54,
95
+ "γ…‘": 0,
96
+ "γ…’": 101,
97
+ "γ…“": 49,
98
+ "γ…”": 45,
99
+ "γ…•": 75,
100
+ "γ…–": 106,
101
+ "γ…—": 63,
102
+ "γ…˜": 10,
103
+ "γ…™": 104,
104
+ "γ…š": 6,
105
+ "γ…›": 3,
106
+ "γ…œ": 1,
107
+ "ㅝ": 99,
108
+ "γ…ž": 18,
109
+ "γ…Ÿ": 30,
110
+ "γ… ": 28,
111
+ "γ…‘": 5,
112
+ "γ…’": 65,
113
+ "γ…£": 73
114
  }