Hezar: Upload tokenizer_config.yaml
Browse files
preprocessor/tokenizer_config.yaml
CHANGED
|
@@ -15,18 +15,6 @@ sep_token: <sep>
|
|
| 15 |
pad_token: </s>
|
| 16 |
cls_token: <cls>
|
| 17 |
mask_token: <mask>
|
| 18 |
-
special_tokens:
|
| 19 |
-
- <s>
|
| 20 |
-
- <pad>
|
| 21 |
-
- </s>
|
| 22 |
-
- <unk>
|
| 23 |
-
- <mask>
|
| 24 |
-
- <|endoftext|>
|
| 25 |
-
- <|startoftext|>
|
| 26 |
-
- <nl>
|
| 27 |
-
- <hs>
|
| 28 |
-
- <sep>
|
| 29 |
-
- <cls>
|
| 30 |
continuing_subword_prefix: ''
|
| 31 |
end_of_word_suffix: ''
|
| 32 |
fuse_unk: false
|
|
|
|
| 15 |
pad_token: </s>
|
| 16 |
cls_token: <cls>
|
| 17 |
mask_token: <mask>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
continuing_subword_prefix: ''
|
| 19 |
end_of_word_suffix: ''
|
| 20 |
fuse_unk: false
|