Hugging Face's logo

llmware
/
stablelm-2-zephyr-1_6b-ov

OpenVINO
stablelm
green
p1
llmware-chat
ov
Model card Files Community
doberst commited on
Commit
0b0ff67
·
verified ·
1 Parent(s): da8fee9

Upload 13 files

Browse files
Files changed (13) hide show
  1. config.json +30 -0
  2. generation_config.json +6 -0
  3. hash_record_sha256.json +9 -0
  4. merges.txt +0 -0
  5. openvino_detokenizer.bin +3 -0
  6. openvino_detokenizer.xml +219 -0
  7. openvino_model.xml +0 -0
  8. openvino_tokenizer.bin +3 -0
  9. openvino_tokenizer.xml +564 -0
  10. special_tokens_map.json +65 -0
  11. tokenizer.json +0 -0
  12. tokenizer_config.json +312 -0
  13. vocab.json +0 -0
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "stabilityai/stablelm-2-zephyr-1_6b",
3
+ "architectures": [
4
+ "StableLmForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 100257,
8
+ "eos_token_id": 100257,
9
+ "hidden_act": "silu",
10
+ "hidden_dropout": 0.0,
11
+ "hidden_size": 2048,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 5632,
14
+ "layer_norm_eps": 1e-05,
15
+ "max_position_embeddings": 4096,
16
+ "model_type": "stablelm",
17
+ "num_attention_heads": 32,
18
+ "num_hidden_layers": 24,
19
+ "num_key_value_heads": 32,
20
+ "partial_rotary_factor": 0.25,
21
+ "qk_layernorm": false,
22
+ "rope_scaling": null,
23
+ "rope_theta": 10000,
24
+ "tie_word_embeddings": false,
25
+ "transformers_version": "4.41.2",
26
+ "use_cache": true,
27
+ "use_parallel_residual": false,
28
+ "use_qkv_bias": true,
29
+ "vocab_size": 100352
30
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 100257,
4
+ "eos_token_id": 100257,
5
+ "transformers_version": "4.41.2"
6
+ }
hash_record_sha256.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "openvino_detokenizer.bin": "14feb27c926213cf4f0398691ab7c467f1bb24f1e9215537e80b05841abf589f",
3
+ "openvino_detokenizer.xml": "33f9af73d5fa5f125473aa2a233da966c710b805270dacbdcd8682b015c47a18",
4
+ "openvino_model.bin": "a8c2303b49fbd8ff8b4d29778ee9d7098ee2e98323125b2f75a8bd2512e6f297",
5
+ "openvino_model.xml": "8f9d873769af98bf172840e2b6f8f990beb09b0856ae90e27adb97644f661ba2",
6
+ "openvino_tokenizer.bin": "ce06e9fec1aaa5f3bd62c983a6e07f85a0a31e63882b0e23721e18020df0950b",
7
+ "openvino_tokenizer.xml": "57e1922829c9fa5dbe2834b23c27a5a0582b200a7b346102133489c8175b7372",
8
+ "time_stamp": "2024-10-07_035806"
9
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
openvino_detokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14feb27c926213cf4f0398691ab7c467f1bb24f1e9215537e80b05841abf589f
3
+ size 1118648
openvino_detokenizer.xml ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0"?>
2
+ <net name="detokenizer" version="11">
3
+ <layers>
4
+ <layer id="0" name="Parameter_192388" type="Parameter" version="opset1">
5
+ <data shape="?,?" element_type="i64" />
6
+ <output>
7
+ <port id="0" precision="I64" names="Parameter_192388">
8
+ <dim>-1</dim>
9
+ <dim>-1</dim>
10
+ </port>
11
+ </output>
12
+ </layer>
13
+ <layer id="1" name="Convert_192404" type="Convert" version="opset1">
14
+ <data destination_type="i32" />
15
+ <input>
16
+ <port id="0" precision="I64">
17
+ <dim>-1</dim>
18
+ <dim>-1</dim>
19
+ </port>
20
+ </input>
21
+ <output>
22
+ <port id="1" precision="I32">
23
+ <dim>-1</dim>
24
+ <dim>-1</dim>
25
+ </port>
26
+ </output>
27
+ </layer>
28
+ <layer id="2" name="Constant_192370" type="Const" version="opset1">
29
+ <data element_type="u8" shape="1118599" offset="0" size="1118599" />
30
+ <output>
31
+ <port id="0" precision="U8">
32
+ <dim>1118599</dim>
33
+ </port>
34
+ </output>
35
+ </layer>
36
+ <layer id="3" name="StringTensorUnpack_192371" type="StringTensorUnpack" version="extension">
37
+ <data mode="begins_ends" />
38
+ <input>
39
+ <port id="0" precision="U8">
40
+ <dim>1118599</dim>
41
+ </port>
42
+ </input>
43
+ <output>
44
+ <port id="1" precision="I32">
45
+ <dim>-1</dim>
46
+ </port>
47
+ <port id="2" precision="I32">
48
+ <dim>-1</dim>
49
+ </port>
50
+ <port id="3" precision="U8">
51
+ <dim>-1</dim>
52
+ </port>
53
+ </output>
54
+ </layer>
55
+ <layer id="4" name="VocabDecoder_192389" type="VocabDecoder" version="extension">
56
+ <data skip_tokens="100256, 100257, 100258, 100259, 100260, 100261, 100262, 100263, 100264, 100265, 100266, 100267, 100268, 100269, 100270, 100271, 100272, 100273, 100274, 100275, 100276, 100277, 100278, 100279, 100280, 100281, 100282, 100283, 100284, 100285, 100286, 100287, 100288" />
57
+ <input>
58
+ <port id="0" precision="I32">
59
+ <dim>-1</dim>
60
+ <dim>-1</dim>
61
+ </port>
62
+ <port id="1" precision="I32">
63
+ <dim>-1</dim>
64
+ </port>
65
+ <port id="2" precision="I32">
66
+ <dim>-1</dim>
67
+ </port>
68
+ <port id="3" precision="U8">
69
+ <dim>-1</dim>
70
+ </port>
71
+ </input>
72
+ <output>
73
+ <port id="4" precision="I32">
74
+ <dim>-1</dim>
75
+ </port>
76
+ <port id="5" precision="I32">
77
+ <dim>-1</dim>
78
+ </port>
79
+ <port id="6" precision="I32">
80
+ <dim>-1</dim>
81
+ </port>
82
+ <port id="7" precision="I32">
83
+ <dim>-1</dim>
84
+ </port>
85
+ <port id="8" precision="U8">
86
+ <dim>-1</dim>
87
+ </port>
88
+ </output>
89
+ </layer>
90
+ <layer id="5" name="CharsToBytes_192390" type="CharsToBytes" version="extension">
91
+ <input>
92
+ <port id="0" precision="I32">
93
+ <dim>-1</dim>
94
+ </port>
95
+ <port id="1" precision="I32">
96
+ <dim>-1</dim>
97
+ </port>
98
+ <port id="2" precision="I32">
99
+ <dim>-1</dim>
100
+ </port>
101
+ <port id="3" precision="I32">
102
+ <dim>-1</dim>
103
+ </port>
104
+ <port id="4" precision="U8">
105
+ <dim>-1</dim>
106
+ </port>
107
+ </input>
108
+ <output>
109
+ <port id="5" precision="I32">
110
+ <dim>-1</dim>
111
+ </port>
112
+ <port id="6" precision="I32">
113
+ <dim>-1</dim>
114
+ </port>
115
+ <port id="7" precision="U8">
116
+ <dim>-1</dim>
117
+ </port>
118
+ </output>
119
+ </layer>
120
+ <layer id="6" name="Constant_192392" type="Const" version="opset1">
121
+ <data element_type="u8" shape="47" offset="1118599" size="47" />
122
+ <output>
123
+ <port id="0" precision="U8">
124
+ <dim>47</dim>
125
+ </port>
126
+ </output>
127
+ </layer>
128
+ <layer id="7" name="Constant_192394" type="Const" version="opset1">
129
+ <data element_type="u8" shape="2" offset="1118646" size="2" />
130
+ <output>
131
+ <port id="0" precision="U8">
132
+ <dim>2</dim>
133
+ </port>
134
+ </output>
135
+ </layer>
136
+ <layer id="8" name="RegexNormalization_192395" type="RegexNormalization" version="extension">
137
+ <data global_replace="true" />
138
+ <input>
139
+ <port id="0" precision="I32">
140
+ <dim>-1</dim>
141
+ </port>
142
+ <port id="1" precision="I32">
143
+ <dim>-1</dim>
144
+ </port>
145
+ <port id="2" precision="U8">
146
+ <dim>-1</dim>
147
+ </port>
148
+ <port id="3" precision="U8">
149
+ <dim>47</dim>
150
+ </port>
151
+ <port id="4" precision="U8">
152
+ <dim>2</dim>
153
+ </port>
154
+ </input>
155
+ <output>
156
+ <port id="5" precision="I32">
157
+ <dim>-1</dim>
158
+ </port>
159
+ <port id="6" precision="I32">
160
+ <dim>-1</dim>
161
+ </port>
162
+ <port id="7" precision="U8">
163
+ <dim>-1</dim>
164
+ </port>
165
+ </output>
166
+ </layer>
167
+ <layer id="9" name="StringTensorPack_192396" type="StringTensorPack" version="extension">
168
+ <data mode="begins_ends" />
169
+ <input>
170
+ <port id="0" precision="I32">
171
+ <dim>-1</dim>
172
+ </port>
173
+ <port id="1" precision="I32">
174
+ <dim>-1</dim>
175
+ </port>
176
+ <port id="2" precision="U8">
177
+ <dim>-1</dim>
178
+ </port>
179
+ </input>
180
+ <output>
181
+ <port id="3" precision="STRING" names="string_output">
182
+ <dim>-1</dim>
183
+ </port>
184
+ </output>
185
+ </layer>
186
+ <layer id="10" name="Result_192397" type="Result" version="opset1">
187
+ <input>
188
+ <port id="0" precision="STRING">
189
+ <dim>-1</dim>
190
+ </port>
191
+ </input>
192
+ </layer>
193
+ </layers>
194
+ <edges>
195
+ <edge from-layer="0" from-port="0" to-layer="1" to-port="0" />
196
+ <edge from-layer="1" from-port="1" to-layer="4" to-port="0" />
197
+ <edge from-layer="2" from-port="0" to-layer="3" to-port="0" />
198
+ <edge from-layer="3" from-port="1" to-layer="4" to-port="1" />
199
+ <edge from-layer="3" from-port="2" to-layer="4" to-port="2" />
200
+ <edge from-layer="3" from-port="3" to-layer="4" to-port="3" />
201
+ <edge from-layer="4" from-port="4" to-layer="5" to-port="0" />
202
+ <edge from-layer="4" from-port="5" to-layer="5" to-port="1" />
203
+ <edge from-layer="4" from-port="6" to-layer="5" to-port="2" />
204
+ <edge from-layer="4" from-port="7" to-layer="5" to-port="3" />
205
+ <edge from-layer="4" from-port="8" to-layer="5" to-port="4" />
206
+ <edge from-layer="5" from-port="5" to-layer="8" to-port="0" />
207
+ <edge from-layer="5" from-port="6" to-layer="8" to-port="1" />
208
+ <edge from-layer="5" from-port="7" to-layer="8" to-port="2" />
209
+ <edge from-layer="6" from-port="0" to-layer="8" to-port="3" />
210
+ <edge from-layer="7" from-port="0" to-layer="8" to-port="4" />
211
+ <edge from-layer="8" from-port="5" to-layer="9" to-port="0" />
212
+ <edge from-layer="8" from-port="6" to-layer="9" to-port="1" />
213
+ <edge from-layer="8" from-port="7" to-layer="9" to-port="2" />
214
+ <edge from-layer="9" from-port="3" to-layer="10" to-port="0" />
215
+ </edges>
216
+ <rt_info>
217
+ <eos_token_id value="100257" />
218
+ </rt_info>
219
+ </net>
openvino_model.xml ADDED
The diff for this file is too large to render. See raw diff
 
openvino_tokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce06e9fec1aaa5f3bd62c983a6e07f85a0a31e63882b0e23721e18020df0950b
3
+ size 2335372
openvino_tokenizer.xml ADDED
@@ -0,0 +1,564 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0"?>
2
+ <net name="tokenizer" version="11">
3
+ <layers>
4
+ <layer id="0" name="Parameter_192289" type="Parameter" version="opset1">
5
+ <data shape="?" element_type="string" />
6
+ <output>
7
+ <port id="0" precision="STRING" names="Parameter_192289">
8
+ <dim>-1</dim>
9
+ </port>
10
+ </output>
11
+ </layer>
12
+ <layer id="1" name="Constant_192295" type="Const" version="opset1">
13
+ <data element_type="i32" shape="" offset="0" size="4" />
14
+ <output>
15
+ <port id="0" precision="I32" />
16
+ </output>
17
+ </layer>
18
+ <layer id="2" name="StringTensorUnpack_192290" type="StringTensorUnpack" version="extension">
19
+ <data mode="begins_ends" />
20
+ <input>
21
+ <port id="0" precision="STRING">
22
+ <dim>-1</dim>
23
+ </port>
24
+ </input>
25
+ <output>
26
+ <port id="1" precision="I32">
27
+ <dim>-1</dim>
28
+ </port>
29
+ <port id="2" precision="I32">
30
+ <dim>-1</dim>
31
+ </port>
32
+ <port id="3" precision="U8">
33
+ <dim>-1</dim>
34
+ </port>
35
+ </output>
36
+ </layer>
37
+ <layer id="3" name="ShapeOf_192291" type="ShapeOf" version="opset3">
38
+ <data output_type="i64" />
39
+ <input>
40
+ <port id="0" precision="I32">
41
+ <dim>-1</dim>
42
+ </port>
43
+ </input>
44
+ <output>
45
+ <port id="1" precision="I64">
46
+ <dim>1</dim>
47
+ </port>
48
+ </output>
49
+ </layer>
50
+ <layer id="4" name="Constant_192292" type="Const" version="opset1">
51
+ <data element_type="i32" shape="" offset="0" size="4" />
52
+ <output>
53
+ <port id="0" precision="I32" />
54
+ </output>
55
+ </layer>
56
+ <layer id="5" name="Constant_192293" type="Const" version="opset1">
57
+ <data element_type="i32" shape="" offset="0" size="4" />
58
+ <output>
59
+ <port id="0" precision="I32" />
60
+ </output>
61
+ </layer>
62
+ <layer id="6" name="Gather_192294" type="Gather" version="opset8">
63
+ <data batch_dims="0" />
64
+ <input>
65
+ <port id="0" precision="I64">
66
+ <dim>1</dim>
67
+ </port>
68
+ <port id="1" precision="I32" />
69
+ <port id="2" precision="I32" />
70
+ </input>
71
+ <output>
72
+ <port id="3" precision="I64" />
73
+ </output>
74
+ </layer>
75
+ <layer id="7" name="Constant_192296" type="Const" version="opset1">
76
+ <data element_type="i32" shape="" offset="4" size="4" />
77
+ <output>
78
+ <port id="0" precision="I32" />
79
+ </output>
80
+ </layer>
81
+ <layer id="8" name="Range_192297" type="Range" version="opset4">
82
+ <data output_type="i32" />
83
+ <input>
84
+ <port id="0" precision="I32" />
85
+ <port id="1" precision="I64" />
86
+ <port id="2" precision="I32" />
87
+ </input>
88
+ <output>
89
+ <port id="3" precision="I32">
90
+ <dim>-1</dim>
91
+ </port>
92
+ </output>
93
+ </layer>
94
+ <layer id="9" name="Constant_192299" type="Const" version="opset1">
95
+ <data element_type="i32" shape="" offset="4" size="4" />
96
+ <output>
97
+ <port id="0" precision="I32" />
98
+ </output>
99
+ </layer>
100
+ <layer id="10" name="Constant_192300" type="Const" version="opset1">
101
+ <data element_type="i64" shape="" offset="8" size="8" />
102
+ <output>
103
+ <port id="0" precision="I64" />
104
+ </output>
105
+ </layer>
106
+ <layer id="11" name="Add_192301" type="Add" version="opset1">
107
+ <data auto_broadcast="numpy" />
108
+ <input>
109
+ <port id="0" precision="I64" />
110
+ <port id="1" precision="I64" />
111
+ </input>
112
+ <output>
113
+ <port id="2" precision="I64" />
114
+ </output>
115
+ </layer>
116
+ <layer id="12" name="Constant_192302" type="Const" version="opset1">
117
+ <data element_type="i32" shape="" offset="4" size="4" />
118
+ <output>
119
+ <port id="0" precision="I32" />
120
+ </output>
121
+ </layer>
122
+ <layer id="13" name="Range_192303" type="Range" version="opset4">
123
+ <data output_type="i32" />
124
+ <input>
125
+ <port id="0" precision="I32" />
126
+ <port id="1" precision="I64" />
127
+ <port id="2" precision="I32" />
128
+ </input>
129
+ <output>
130
+ <port id="3" precision="I32">
131
+ <dim>-1</dim>
132
+ </port>
133
+ </output>
134
+ </layer>
135
+ <layer id="14" name="Constant_192366" type="Const" version="opset1">
136
+ <data element_type="u8" shape="109" offset="16" size="109" />
137
+ <output>
138
+ <port id="0" precision="U8">
139
+ <dim>109</dim>
140
+ </port>
141
+ </output>
142
+ </layer>
143
+ <layer id="15" name="RegexSplit_192367" type="RegexSplit" version="extension">
144
+ <data behaviour="remove" invert="true" max_splits="-1" />
145
+ <input>
146
+ <port id="0" precision="I32">
147
+ <dim>-1</dim>
148
+ </port>
149
+ <port id="1" precision="I32">
150
+ <dim>-1</dim>
151
+ </port>
152
+ <port id="2" precision="I32">
153
+ <dim>-1</dim>
154
+ </port>
155
+ <port id="3" precision="I32">
156
+ <dim>-1</dim>
157
+ </port>
158
+ <port id="4" precision="U8">
159
+ <dim>-1</dim>
160
+ </port>
161
+ <port id="5" precision="U8">
162
+ <dim>109</dim>
163
+ </port>
164
+ </input>
165
+ <output>
166
+ <port id="6" precision="I32">
167
+ <dim>-1</dim>
168
+ </port>
169
+ <port id="7" precision="I32">
170
+ <dim>-1</dim>
171
+ </port>
172
+ <port id="8" precision="I32">
173
+ <dim>-1</dim>
174
+ </port>
175
+ <port id="9" precision="I32">
176
+ <dim>-1</dim>
177
+ </port>
178
+ <port id="10" precision="U8">
179
+ <dim>-1</dim>
180
+ </port>
181
+ </output>
182
+ </layer>
183
+ <layer id="16" name="BytesToChars_192368" type="BytesToChars" version="extension">
184
+ <input>
185
+ <port id="0" precision="I32">
186
+ <dim>-1</dim>
187
+ </port>
188
+ <port id="1" precision="I32">
189
+ <dim>-1</dim>
190
+ </port>
191
+ <port id="2" precision="I32">
192
+ <dim>-1</dim>
193
+ </port>
194
+ <port id="3" precision="I32">
195
+ <dim>-1</dim>
196
+ </port>
197
+ <port id="4" precision="U8">
198
+ <dim>-1</dim>
199
+ </port>
200
+ </input>
201
+ <output>
202
+ <port id="5" precision="I32">
203
+ <dim>-1</dim>
204
+ </port>
205
+ <port id="6" precision="I32">
206
+ <dim>-1</dim>
207
+ </port>
208
+ <port id="7" precision="I32">
209
+ <dim>-1</dim>
210
+ </port>
211
+ <port id="8" precision="I32">
212
+ <dim>-1</dim>
213
+ </port>
214
+ <port id="9" precision="U8">
215
+ <dim>-1</dim>
216
+ </port>
217
+ </output>
218
+ </layer>
219
+ <layer id="17" name="Constant_192370" type="Const" version="opset1">
220
+ <data element_type="u8" shape="1118599" offset="125" size="1118599" />
221
+ <output>
222
+ <port id="0" precision="U8">
223
+ <dim>1118599</dim>
224
+ </port>
225
+ </output>
226
+ </layer>
227
+ <layer id="18" name="StringTensorUnpack_192371" type="StringTensorUnpack" version="extension">
228
+ <data mode="begins_ends" />
229
+ <input>
230
+ <port id="0" precision="U8">
231
+ <dim>1118599</dim>
232
+ </port>
233
+ </input>
234
+ <output>
235
+ <port id="1" precision="I32">
236
+ <dim>-1</dim>
237
+ </port>
238
+ <port id="2" precision="I32">
239
+ <dim>-1</dim>
240
+ </port>
241
+ <port id="3" precision="U8">
242
+ <dim>-1</dim>
243
+ </port>
244
+ </output>
245
+ </layer>
246
+ <layer id="19" name="Constant_192373" type="Const" version="opset1">
247
+ <data element_type="u8" shape="1216640" offset="1118724" size="1216640" />
248
+ <output>
249
+ <port id="0" precision="U8">
250
+ <dim>1216640</dim>
251
+ </port>
252
+ </output>
253
+ </layer>
254
+ <layer id="20" name="StringTensorUnpack_192374" type="StringTensorUnpack" version="extension">
255
+ <data mode="begins_ends" />
256
+ <input>
257
+ <port id="0" precision="U8">
258
+ <dim>1216640</dim>
259
+ </port>
260
+ </input>
261
+ <output>
262
+ <port id="1" precision="I32">
263
+ <dim>-1</dim>
264
+ </port>
265
+ <port id="2" precision="I32">
266
+ <dim>-1</dim>
267
+ </port>
268
+ <port id="3" precision="U8">
269
+ <dim>-1</dim>
270
+ </port>
271
+ </output>
272
+ </layer>
273
+ <layer id="21" name="BPETokenizer_192375" type="BPETokenizer" version="extension">
274
+ <data unk_token="" fuse_unk="false" suffix_indicator="" end_suffix="" byte_fallback="false" />
275
+ <input>
276
+ <port id="0" precision="I32">
277
+ <dim>-1</dim>
278
+ </port>
279
+ <port id="1" precision="I32">
280
+ <dim>-1</dim>
281
+ </port>
282
+ <port id="2" precision="I32">
283
+ <dim>-1</dim>
284
+ </port>
285
+ <port id="3" precision="I32">
286
+ <dim>-1</dim>
287
+ </port>
288
+ <port id="4" precision="U8">
289
+ <dim>-1</dim>
290
+ </port>
291
+ <port id="5" precision="I32">
292
+ <dim>-1</dim>
293
+ </port>
294
+ <port id="6" precision="I32">
295
+ <dim>-1</dim>
296
+ </port>
297
+ <port id="7" precision="U8">
298
+ <dim>-1</dim>
299
+ </port>
300
+ <port id="8" precision="I32">
301
+ <dim>-1</dim>
302
+ </port>
303
+ <port id="9" precision="I32">
304
+ <dim>-1</dim>
305
+ </port>
306
+ <port id="10" precision="U8">
307
+ <dim>-1</dim>
308
+ </port>
309
+ </input>
310
+ <output>
311
+ <port id="11" precision="I32">
312
+ <dim>-1</dim>
313
+ </port>
314
+ <port id="12" precision="I32">
315
+ <dim>-1</dim>
316
+ </port>
317
+ <port id="13" precision="I32">
318
+ <dim>-1</dim>
319
+ </port>
320
+ </output>
321
+ </layer>
322
+ <layer id="22" name="Subtract_192376" type="Subtract" version="opset1">
323
+ <data auto_broadcast="numpy" />
324
+ <input>
325
+ <port id="0" precision="I32">
326
+ <dim>-1</dim>
327
+ </port>
328
+ <port id="1" precision="I32">
329
+ <dim>-1</dim>
330
+ </port>
331
+ </input>
332
+ <output>
333
+ <port id="2" precision="I32">
334
+ <dim>-1</dim>
335
+ </port>
336
+ </output>
337
+ </layer>
338
+ <layer id="23" name="Constant_192377" type="Const" version="opset1">
339
+ <data element_type="i32" shape="" offset="2335364" size="4" />
340
+ <output>
341
+ <port id="0" precision="I32" />
342
+ </output>
343
+ </layer>
344
+ <layer id="24" name="Minimum_192378" type="Minimum" version="opset1">
345
+ <data auto_broadcast="numpy" />
346
+ <input>
347
+ <port id="0" precision="I32">
348
+ <dim>-1</dim>
349
+ </port>
350
+ <port id="1" precision="I32" />
351
+ </input>
352
+ <output>
353
+ <port id="2" precision="I32">
354
+ <dim>-1</dim>
355
+ </port>
356
+ </output>
357
+ </layer>
358
+ <layer id="25" name="Add_192379" type="Add" version="opset1">
359
+ <data auto_broadcast="numpy" />
360
+ <input>
361
+ <port id="0" precision="I32">
362
+ <dim>-1</dim>
363
+ </port>
364
+ <port id="1" precision="I32">
365
+ <dim>-1</dim>
366
+ </port>
367
+ </input>
368
+ <output>
369
+ <port id="2" precision="I32">
370
+ <dim>-1</dim>
371
+ </port>
372
+ </output>
373
+ </layer>
374
+ <layer id="26" name="Subtract_192380" type="Subtract" version="opset1">
375
+ <data auto_broadcast="numpy" />
376
+ <input>
377
+ <port id="0" precision="I32">
378
+ <dim>-1</dim>
379
+ </port>
380
+ <port id="1" precision="I32">
381
+ <dim>-1</dim>
382
+ </port>
383
+ </input>
384
+ <output>
385
+ <port id="2" precision="I32">
386
+ <dim>-1</dim>
387
+ </port>
388
+ </output>
389
+ </layer>
390
+ <layer id="27" name="Constant_192381" type="Const" version="opset1">
391
+ <data element_type="i32" shape="" offset="0" size="4" />
392
+ <output>
393
+ <port id="0" precision="I32" />
394
+ </output>
395
+ </layer>
396
+ <layer id="28" name="ReduceMax_192382" type="ReduceMax" version="opset1">
397
+ <data keep_dims="false" />
398
+ <input>
399
+ <port id="0" precision="I32">
400
+ <dim>-1</dim>
401
+ </port>
402
+ <port id="1" precision="I32" />
403
+ </input>
404
+ <output>
405
+ <port id="2" precision="I32" />
406
+ </output>
407
+ </layer>
408
+ <layer id="29" name="Constant_192383" type="Const" version="opset1">
409
+ <data element_type="i32" shape="" offset="2335368" size="4" />
410
+ <output>
411
+ <port id="0" precision="I32" />
412
+ </output>
413
+ </layer>
414
+ <layer id="30" name="RaggedToDense_192384" type="RaggedToDense" version="extension">
415
+ <data pad_right="true" />
416
+ <input>
417
+ <port id="0" precision="I32">
418
+ <dim>-1</dim>
419
+ </port>
420
+ <port id="1" precision="I32">
421
+ <dim>-1</dim>
422
+ </port>
423
+ <port id="2" precision="I32">
424
+ <dim>-1</dim>
425
+ </port>
426
+ <port id="3" precision="I32" />
427
+ <port id="4" precision="I32" />
428
+ </input>
429
+ <output>
430
+ <port id="5" precision="I32">
431
+ <dim>-1</dim>
432
+ <dim>-1</dim>
433
+ </port>
434
+ <port id="6" precision="BOOL">
435
+ <dim>-1</dim>
436
+ <dim>-1</dim>
437
+ </port>
438
+ </output>
439
+ </layer>
440
+ <layer id="31" name="Convert_192385" type="Convert" version="opset1">
441
+ <data destination_type="i32" />
442
+ <input>
443
+ <port id="0" precision="BOOL">
444
+ <dim>-1</dim>
445
+ <dim>-1</dim>
446
+ </port>
447
+ </input>
448
+ <output>
449
+ <port id="1" precision="I32">
450
+ <dim>-1</dim>
451
+ <dim>-1</dim>
452
+ </port>
453
+ </output>
454
+ </layer>
455
+ <layer id="32" name="Convert_192385" type="Convert" version="opset1">
456
+ <data destination_type="i64" />
457
+ <input>
458
+ <port id="0" precision="I32">
459
+ <dim>-1</dim>
460
+ <dim>-1</dim>
461
+ </port>
462
+ </input>
463
+ <output>
464
+ <port id="1" precision="I64" names="attention_mask">
465
+ <dim>-1</dim>
466
+ <dim>-1</dim>
467
+ </port>
468
+ </output>
469
+ </layer>
470
+ <layer id="34" name="RaggedToDense_192384.0" type="Convert" version="opset1">
471
+ <data destination_type="i64" />
472
+ <input>
473
+ <port id="0" precision="I32">
474
+ <dim>-1</dim>
475
+ <dim>-1</dim>
476
+ </port>
477
+ </input>
478
+ <output>
479
+ <port id="1" precision="I64" names="input_ids">
480
+ <dim>-1</dim>
481
+ <dim>-1</dim>
482
+ </port>
483
+ </output>
484
+ </layer>
485
+ <layer id="35" name="Result_192386" type="Result" version="opset1">
486
+ <input>
487
+ <port id="0" precision="I64">
488
+ <dim>-1</dim>
489
+ <dim>-1</dim>
490
+ </port>
491
+ </input>
492
+ </layer>
493
+ <layer id="33" name="Result_192387" type="Result" version="opset1">
494
+ <input>
495
+ <port id="0" precision="I64">
496
+ <dim>-1</dim>
497
+ <dim>-1</dim>
498
+ </port>
499
+ </input>
500
+ </layer>
501
+ </layers>
502
+ <edges>
503
+ <edge from-layer="0" from-port="0" to-layer="2" to-port="0" />
504
+ <edge from-layer="1" from-port="0" to-layer="8" to-port="0" />
505
+ <edge from-layer="2" from-port="1" to-layer="3" to-port="0" />
506
+ <edge from-layer="2" from-port="1" to-layer="15" to-port="2" />
507
+ <edge from-layer="2" from-port="2" to-layer="15" to-port="3" />
508
+ <edge from-layer="2" from-port="3" to-layer="15" to-port="4" />
509
+ <edge from-layer="3" from-port="1" to-layer="6" to-port="0" />
510
+ <edge from-layer="4" from-port="0" to-layer="6" to-port="1" />
511
+ <edge from-layer="5" from-port="0" to-layer="6" to-port="2" />
512
+ <edge from-layer="6" from-port="3" to-layer="8" to-port="1" />
513
+ <edge from-layer="6" from-port="3" to-layer="11" to-port="0" />
514
+ <edge from-layer="7" from-port="0" to-layer="8" to-port="2" />
515
+ <edge from-layer="8" from-port="3" to-layer="15" to-port="0" />
516
+ <edge from-layer="9" from-port="0" to-layer="13" to-port="0" />
517
+ <edge from-layer="10" from-port="0" to-layer="11" to-port="1" />
518
+ <edge from-layer="11" from-port="2" to-layer="13" to-port="1" />
519
+ <edge from-layer="12" from-port="0" to-layer="13" to-port="2" />
520
+ <edge from-layer="13" from-port="3" to-layer="15" to-port="1" />
521
+ <edge from-layer="14" from-port="0" to-layer="15" to-port="5" />
522
+ <edge from-layer="15" from-port="6" to-layer="16" to-port="0" />
523
+ <edge from-layer="15" from-port="7" to-layer="16" to-port="1" />
524
+ <edge from-layer="15" from-port="8" to-layer="16" to-port="2" />
525
+ <edge from-layer="15" from-port="9" to-layer="16" to-port="3" />
526
+ <edge from-layer="15" from-port="10" to-layer="16" to-port="4" />
527
+ <edge from-layer="16" from-port="8" to-layer="21" to-port="3" />
528
+ <edge from-layer="16" from-port="9" to-layer="21" to-port="4" />
529
+ <edge from-layer="16" from-port="5" to-layer="21" to-port="0" />
530
+ <edge from-layer="16" from-port="6" to-layer="21" to-port="1" />
531
+ <edge from-layer="16" from-port="7" to-layer="21" to-port="2" />
532
+ <edge from-layer="17" from-port="0" to-layer="18" to-port="0" />
533
+ <edge from-layer="18" from-port="1" to-layer="21" to-port="5" />
534
+ <edge from-layer="18" from-port="2" to-layer="21" to-port="6" />
535
+ <edge from-layer="18" from-port="3" to-layer="21" to-port="7" />
536
+ <edge from-layer="19" from-port="0" to-layer="20" to-port="0" />
537
+ <edge from-layer="20" from-port="1" to-layer="21" to-port="8" />
538
+ <edge from-layer="20" from-port="2" to-layer="21" to-port="9" />
539
+ <edge from-layer="20" from-port="3" to-layer="21" to-port="10" />
540
+ <edge from-layer="21" from-port="12" to-layer="22" to-port="0" />
541
+ <edge from-layer="21" from-port="11" to-layer="22" to-port="1" />
542
+ <edge from-layer="21" from-port="11" to-layer="25" to-port="0" />
543
+ <edge from-layer="21" from-port="13" to-layer="30" to-port="2" />
544
+ <edge from-layer="21" from-port="11" to-layer="26" to-port="1" />
545
+ <edge from-layer="21" from-port="11" to-layer="30" to-port="0" />
546
+ <edge from-layer="22" from-port="2" to-layer="24" to-port="0" />
547
+ <edge from-layer="23" from-port="0" to-layer="24" to-port="1" />
548
+ <edge from-layer="24" from-port="2" to-layer="25" to-port="1" />
549
+ <edge from-layer="25" from-port="2" to-layer="26" to-port="0" />
550
+ <edge from-layer="25" from-port="2" to-layer="30" to-port="1" />
551
+ <edge from-layer="26" from-port="2" to-layer="28" to-port="0" />
552
+ <edge from-layer="27" from-port="0" to-layer="28" to-port="1" />
553
+ <edge from-layer="28" from-port="2" to-layer="30" to-port="3" />
554
+ <edge from-layer="29" from-port="0" to-layer="30" to-port="4" />
555
+ <edge from-layer="30" from-port="6" to-layer="31" to-port="0" />
556
+ <edge from-layer="30" from-port="5" to-layer="34" to-port="0" />
557
+ <edge from-layer="31" from-port="1" to-layer="32" to-port="0" />
558
+ <edge from-layer="32" from-port="1" to-layer="33" to-port="0" />
559
+ <edge from-layer="34" from-port="1" to-layer="35" to-port="0" />
560
+ </edges>
561
+ <rt_info>
562
+ <eos_token_id value="100257" />
563
+ </rt_info>
564
+ </net>
special_tokens_map.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|reg_extra|>",
4
+ "<|endoftext|>",
5
+ "<|fim_prefix|>",
6
+ "<|fim_middle|>",
7
+ "<|fim_suffix|>",
8
+ "<|fim_pad|>",
9
+ "<gh_stars>",
10
+ "<filename>",
11
+ "<issue_start>",
12
+ "<issue_comment>",
13
+ "<issue_closed>",
14
+ "<jupyter_start>",
15
+ "<jupyter_text>",
16
+ "<jupyter_code>",
17
+ "<jupyter_output>",
18
+ "<empty_output>",
19
+ "<commit_before>",
20
+ "<commit_msg>",
21
+ "<commit_after>",
22
+ "<reponame>",
23
+ "<|endofprompt|>",
24
+ "<|im_start|>",
25
+ "<|im_end|>",
26
+ "<|pause|>",
27
+ "<|reg0|>",
28
+ "<|reg1|>",
29
+ "<|reg2|>",
30
+ "<|reg3|>",
31
+ "<|reg4|>",
32
+ "<|reg5|>",
33
+ "<|reg6|>",
34
+ "<|reg7|>",
35
+ "<|extra0|>"
36
+ ],
37
+ "bos_token": {
38
+ "content": "<|endoftext|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
+ "eos_token": {
45
+ "content": "<|endoftext|>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false
50
+ },
51
+ "pad_token": {
52
+ "content": "<|endoftext|>",
53
+ "lstrip": false,
54
+ "normalized": false,
55
+ "rstrip": false,
56
+ "single_word": false
57
+ },
58
+ "unk_token": {
59
+ "content": "<|endoftext|>",
60
+ "lstrip": false,
61
+ "normalized": false,
62
+ "rstrip": false,
63
+ "single_word": false
64
+ }
65
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,312 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "100256": {
5
+ "content": "<|reg_extra|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "100257": {
13
+ "content": "<|endoftext|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "100258": {
21
+ "content": "<|fim_prefix|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "100259": {
29
+ "content": "<|fim_middle|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "100260": {
37
+ "content": "<|fim_suffix|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "100261": {
45
+ "content": "<|fim_pad|>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "100262": {
53
+ "content": "<gh_stars>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "100263": {
61
+ "content": "<filename>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "100264": {
69
+ "content": "<issue_start>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "100265": {
77
+ "content": "<issue_comment>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "100266": {
85
+ "content": "<issue_closed>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "100267": {
93
+ "content": "<jupyter_start>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "100268": {
101
+ "content": "<jupyter_text>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "100269": {
109
+ "content": "<jupyter_code>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "100270": {
117
+ "content": "<jupyter_output>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "100271": {
125
+ "content": "<empty_output>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "100272": {
133
+ "content": "<commit_before>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "100273": {
141
+ "content": "<commit_msg>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "100274": {
149
+ "content": "<commit_after>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "100275": {
157
+ "content": "<reponame>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "100276": {
165
+ "content": "<|endofprompt|>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "100277": {
173
+ "content": "<|im_start|>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "100278": {
181
+ "content": "<|im_end|>",
182
+ "lstrip": false,
183
+ "normalized": false,
184
+ "rstrip": false,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "100279": {
189
+ "content": "<|pause|>",
190
+ "lstrip": false,
191
+ "normalized": false,
192
+ "rstrip": false,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "100280": {
197
+ "content": "<|reg0|>",
198
+ "lstrip": false,
199
+ "normalized": false,
200
+ "rstrip": false,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "100281": {
205
+ "content": "<|reg1|>",
206
+ "lstrip": false,
207
+ "normalized": false,
208
+ "rstrip": false,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "100282": {
213
+ "content": "<|reg2|>",
214
+ "lstrip": false,
215
+ "normalized": false,
216
+ "rstrip": false,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "100283": {
221
+ "content": "<|reg3|>",
222
+ "lstrip": false,
223
+ "normalized": false,
224
+ "rstrip": false,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "100284": {
229
+ "content": "<|reg4|>",
230
+ "lstrip": false,
231
+ "normalized": false,
232
+ "rstrip": false,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "100285": {
237
+ "content": "<|reg5|>",
238
+ "lstrip": false,
239
+ "normalized": false,
240
+ "rstrip": false,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "100286": {
245
+ "content": "<|reg6|>",
246
+ "lstrip": false,
247
+ "normalized": false,
248
+ "rstrip": false,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "100287": {
253
+ "content": "<|reg7|>",
254
+ "lstrip": false,
255
+ "normalized": false,
256
+ "rstrip": false,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "100288": {
261
+ "content": "<|extra0|>",
262
+ "lstrip": false,
263
+ "normalized": false,
264
+ "rstrip": false,
265
+ "single_word": false,
266
+ "special": true
267
+ }
268
+ },
269
+ "additional_special_tokens": [
270
+ "<|reg_extra|>",
271
+ "<|endoftext|>",
272
+ "<|fim_prefix|>",
273
+ "<|fim_middle|>",
274
+ "<|fim_suffix|>",
275
+ "<|fim_pad|>",
276
+ "<gh_stars>",
277
+ "<filename>",
278
+ "<issue_start>",
279
+ "<issue_comment>",
280
+ "<issue_closed>",
281
+ "<jupyter_start>",
282
+ "<jupyter_text>",
283
+ "<jupyter_code>",
284
+ "<jupyter_output>",
285
+ "<empty_output>",
286
+ "<commit_before>",
287
+ "<commit_msg>",
288
+ "<commit_after>",
289
+ "<reponame>",
290
+ "<|endofprompt|>",
291
+ "<|im_start|>",
292
+ "<|im_end|>",
293
+ "<|pause|>",
294
+ "<|reg0|>",
295
+ "<|reg1|>",
296
+ "<|reg2|>",
297
+ "<|reg3|>",
298
+ "<|reg4|>",
299
+ "<|reg5|>",
300
+ "<|reg6|>",
301
+ "<|reg7|>",
302
+ "<|extra0|>"
303
+ ],
304
+ "bos_token": "<|endoftext|>",
305
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
306
+ "clean_up_tokenization_spaces": true,
307
+ "eos_token": "<|endoftext|>",
308
+ "model_max_length": 4096,
309
+ "pad_token": "<|endoftext|>",
310
+ "tokenizer_class": "GPT2Tokenizer",
311
+ "unk_token": "<|endoftext|>"
312
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff