Upload added_tokens.json with huggingface_hub
Browse files- added_tokens.json +16 -0
added_tokens.json
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<eop>": 151334,
|
3 |
+
"<sop>": 151333,
|
4 |
+
"<|assistant|>": 151337,
|
5 |
+
"<|begin_of_image|>": 151339,
|
6 |
+
"<|begin_of_video|>": 151341,
|
7 |
+
"<|end_of_image|>": 151340,
|
8 |
+
"<|end_of_video|>": 151342,
|
9 |
+
"<|endoftext|>": 151329,
|
10 |
+
"<|observation|>": 151338,
|
11 |
+
"<|system|>": 151335,
|
12 |
+
"<|user|>": 151336,
|
13 |
+
"[MASK]": 151330,
|
14 |
+
"[gMASK]": 151331,
|
15 |
+
"[sMASK]": 151332
|
16 |
+
}
|