Thunderbee commited on
Commit
862729a
1 Parent(s): b32f9d8

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +13 -0
tokenizer_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_max_length": 1024,
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|endoftext|>",
6
+ "unk_token": "<|endoftext|>",
7
+ "padding_side": "right",
8
+ "truncation_side": "right",
9
+ "model_input_names": [
10
+ "input_ids",
11
+ "attention_mask"
12
+ ]
13
+ }