Hugging Face's logo

eliebak
/
smoltokenizer

Model card Files Community
eliebak HF Staff commited on
Commit
c969ba6
·
1 Parent(s): 9c4e961

Initial commit of smoltokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +10 -0
  2. special_tokens_map.json +34 -0
  3. tokenizer_config.json +110 -0
added_tokens.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<file_sep>": 264,
3
+ "<fim_middle>": 260,
4
+ "<fim_pad>": 262,
5
+ "<fim_prefix>": 259,
6
+ "<fim_suffix>": 261,
7
+ "<repo_name>": 263,
8
+ "<|im_end|>": 265,
9
+ "<|im_start|>": 266
10
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<fim_prefix>",
5
+ "<fim_middle>",
6
+ "<fim_suffix>",
7
+ "<fim_pad>",
8
+ "<repo_name>",
9
+ "<file_sep>",
10
+ "<|im_end|>",
11
+ "<|im_start|>"
12
+ ],
13
+ "eos_token": {
14
+ "content": "<|endoftext|>",
15
+ "lstrip": true,
16
+ "normalized": true,
17
+ "rstrip": true,
18
+ "single_word": false
19
+ },
20
+ "pad_token": {
21
+ "content": "<pad>",
22
+ "lstrip": true,
23
+ "normalized": true,
24
+ "rstrip": true,
25
+ "single_word": false
26
+ },
27
+ "unk_token": {
28
+ "content": "<unk>",
29
+ "lstrip": true,
30
+ "normalized": true,
31
+ "rstrip": true,
32
+ "single_word": false
33
+ }
34
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<pad>",
5
+ "lstrip": true,
6
+ "normalized": true,
7
+ "rstrip": true,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<|endoftext|>",
13
+ "lstrip": true,
14
+ "normalized": true,
15
+ "rstrip": true,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<unk>",
21
+ "lstrip": true,
22
+ "normalized": true,
23
+ "rstrip": true,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "259": {
28
+ "content": "<fim_prefix>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "260": {
36
+ "content": "<fim_middle>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "261": {
44
+ "content": "<fim_suffix>",
45
+ "lstrip": false,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ },
51
+ "262": {
52
+ "content": "<fim_pad>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false,
57
+ "special": true
58
+ },
59
+ "263": {
60
+ "content": "<repo_name>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false,
65
+ "special": true
66
+ },
67
+ "264": {
68
+ "content": "<file_sep>",
69
+ "lstrip": false,
70
+ "normalized": false,
71
+ "rstrip": false,
72
+ "single_word": false,
73
+ "special": true
74
+ },
75
+ "265": {
76
+ "content": "<|im_end|>",
77
+ "lstrip": false,
78
+ "normalized": false,
79
+ "rstrip": false,
80
+ "single_word": false,
81
+ "special": true
82
+ },
83
+ "266": {
84
+ "content": "<|im_start|>",
85
+ "lstrip": false,
86
+ "normalized": false,
87
+ "rstrip": false,
88
+ "single_word": false,
89
+ "special": true
90
+ }
91
+ },
92
+ "additional_special_tokens": [
93
+ "<|endoftext|>",
94
+ "<fim_prefix>",
95
+ "<fim_middle>",
96
+ "<fim_suffix>",
97
+ "<fim_pad>",
98
+ "<repo_name>",
99
+ "<file_sep>",
100
+ "<|im_end|>",
101
+ "<|im_start|>"
102
+ ],
103
+ "clean_up_tokenization_spaces": true,
104
+ "eos_token": "<|endoftext|>",
105
+ "extra_ids": 0,
106
+ "model_max_length": 1000000000000000019884624838656,
107
+ "pad_token": "<pad>",
108
+ "tokenizer_class": "ByT5Tokenizer",
109
+ "unk_token": "<unk>"
110
+ }