Leonardo6 commited on
Commit
b118453
1 Parent(s): e46a1f9

Update special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +33 -29
special_tokens_map.json CHANGED
@@ -1,30 +1,34 @@
1
  {
2
- "bos_token": {
3
- "content": "<|startoftext|>",
4
- "lstrip": false,
5
- "normalized": true,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|endoftext|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": {
17
- "content": "<|endoftext|>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
- "unk_token": {
24
- "content": "<|endoftext|>",
25
- "lstrip": false,
26
- "normalized": false,
27
- "rstrip": false,
28
- "single_word": false
29
- }
30
- }
 
 
 
 
 
1
  {
2
+ "unk_token": {
3
+ "content": "<|endoftext|>",
4
+ "single_word": False,
5
+ "lstrip": False,
6
+ "rstrip": False,
7
+ "normalized": True,
8
+ "__type": "AddedToken"
9
+ },
10
+ "bos_token": {
11
+ "content": "<|startoftext|>",
12
+ "single_word": False,
13
+ "lstrip": False,
14
+ "rstrip": False,
15
+ "normalized": True,
16
+ "__type": "AddedToken"
17
+ },
18
+ "eos_token": {
19
+ "content": "<|endoftext|>",
20
+ "single_word": False,
21
+ "lstrip": False,
22
+ "rstrip": False,
23
+ "normalized": True,
24
+ "__type": "AddedToken"
25
+ },
26
+ "pad_token": "<|endoftext|>",
27
+ "errors": "replace",
28
+ "add_prefix_space": False,
29
+ "do_lower_case": True,
30
+ "model_max_length": 77,
31
+ "tokenizer_class": "CLIPTokenizer",
32
+ "name_or_path": "Leonardo6/clip-datacomp-12m-16",
33
+ "special_tokens_map_file": "./special_tokens_map.json"
34
+ }