f77777 commited on
Commit
a1e34fc
·
verified ·
1 Parent(s): af1114c

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +68 -68
vocab.json CHANGED
@@ -1,73 +1,73 @@
1
  {
2
- "'": 24,
3
  "[PAD]": 70,
4
  "[UNK]": 69,
5
- "|": 51,
6
- "ം": 43,
7
  "ഃ": 41,
8
- "അ": 2,
9
- "ആ": 18,
10
- "ഇ": 6,
11
- "ഈ": 15,
12
- "ഉ": 36,
13
- "എ": 19,
14
- "ഏ": 7,
15
- "ഐ": 23,
16
- "ഒ": 28,
17
- "ഓ": 65,
18
- "ക": 29,
19
- "ഖ": 45,
20
- "ഗ": 10,
21
- "ഘ": 60,
22
- "ങ": 0,
23
- "ച": 22,
24
- "ഛ": 3,
25
- "ജ": 62,
26
- "ഞ": 59,
27
- "ട": 42,
28
- "ഠ": 44,
29
- "ഡ": 17,
30
- "ഢ": 64,
31
- "ണ": 9,
32
- "ത": 25,
33
- "ഥ": 1,
34
- "ദ": 39,
35
- "ധ": 11,
36
- "ന": 5,
37
- "പ": 57,
38
- "ഫ": 14,
39
- "ബ": 13,
40
- "ഭ": 35,
41
- "മ": 55,
42
- "യ": 52,
43
- "ര": 21,
44
- "റ": 33,
45
- "ല": 4,
46
- "ള": 56,
47
- "ഴ": 46,
48
- "വ": 50,
49
- "ശ": 47,
50
- "ഷ": 16,
51
- "സ": 49,
52
- "ഹ": 68,
53
- "ാ": 61,
54
- "ി": 66,
55
- "ീ": 34,
56
- "ു": 32,
57
- "ൂ": 40,
58
- "ൃ": 53,
59
- "െ": 26,
60
- "േ": 67,
61
- "ൈ": 31,
62
- "ൊ": 20,
63
- "ോ": 27,
64
- "ൌ": 30,
65
- "്": 54,
66
- "ൗ": 37,
67
- "ൺ": 38,
68
- "ൻ": 8,
69
- "ർ": 12,
70
- "ൽ": 63,
71
- "ൾ": 58,
72
- "’": 48
73
  }
 
1
  {
2
+ "'": 38,
3
  "[PAD]": 70,
4
  "[UNK]": 69,
5
+ "|": 9,
6
+ "ം": 46,
7
  "ഃ": 41,
8
+ "അ": 18,
9
+ "ആ": 28,
10
+ "ഇ": 7,
11
+ "ഈ": 14,
12
+ "ഉ": 51,
13
+ "എ": 61,
14
+ "ഏ": 63,
15
+ "ഐ": 33,
16
+ "ഒ": 53,
17
+ "ഓ": 59,
18
+ "ക": 56,
19
+ "ഖ": 48,
20
+ "ഗ": 25,
21
+ "ഘ": 50,
22
+ "ങ": 68,
23
+ "ച": 45,
24
+ "ഛ": 30,
25
+ "ജ": 24,
26
+ "ഞ": 26,
27
+ "ട": 60,
28
+ "ഠ": 20,
29
+ "ഡ": 58,
30
+ "ഢ": 44,
31
+ "ണ": 3,
32
+ "ത": 52,
33
+ "ഥ": 43,
34
+ "ദ": 57,
35
+ "ധ": 65,
36
+ "ന": 64,
37
+ "പ": 42,
38
+ "ഫ": 21,
39
+ "ബ": 4,
40
+ "ഭ": 66,
41
+ "മ": 49,
42
+ "യ": 17,
43
+ "ര": 23,
44
+ "റ": 22,
45
+ "ല": 11,
46
+ "ള": 0,
47
+ "ഴ": 15,
48
+ "വ": 31,
49
+ "ശ": 34,
50
+ "ഷ": 19,
51
+ "സ": 47,
52
+ "ഹ": 36,
53
+ "ാ": 29,
54
+ "ി": 16,
55
+ "ീ": 10,
56
+ "ു": 27,
57
+ "ൂ": 54,
58
+ "ൃ": 6,
59
+ "െ": 35,
60
+ "േ": 32,
61
+ "ൈ": 12,
62
+ "ൊ": 5,
63
+ "ോ": 13,
64
+ "ൌ": 55,
65
+ "്": 37,
66
+ "ൗ": 8,
67
+ "ൺ": 1,
68
+ "ൻ": 2,
69
+ "ർ": 62,
70
+ "ൽ": 40,
71
+ "ൾ": 39,
72
+ "’": 67
73
  }