Hugging Face's logo

EleutherAI
/
Mistral-7B-v0.1-addition-random-standardized

Transformers
Safetensors
Model card Files
xet
Community
davidoj01 commited on
Commit
12c737d
·
verified ·
1 Parent(s): 6c1711e

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -1
  2. tokenizer_config.json +1 -1
tokenizer.json CHANGED
@@ -1,7 +1,7 @@
1
  {
2
  "version": "1.0",
3
  "truncation": {
4
- "direction": "Right",
5
  "max_length": 1024,
6
  "strategy": "LongestFirst",
7
  "stride": 0
@@ -139,6 +139,7 @@
139
  "end_of_word_suffix": null,
140
  "fuse_unk": true,
141
  "byte_fallback": true,
 
142
  "vocab": {
143
  "<unk>": 0,
144
  "<s>": 1,
 
1
  {
2
  "version": "1.0",
3
  "truncation": {
4
+ "direction": "Left",
5
  "max_length": 1024,
6
  "strategy": "LongestFirst",
7
  "stride": 0
 
139
  "end_of_word_suffix": null,
140
  "fuse_unk": true,
141
  "byte_fallback": true,
142
+ "ignore_merges": false,
143
  "vocab": {
144
  "<unk>": 0,
145
  "<s>": 1,
tokenizer_config.json CHANGED
@@ -39,7 +39,7 @@
39
  "spaces_between_special_tokens": false,
40
  "stride": 0,
41
  "tokenizer_class": "LlamaTokenizer",
42
- "truncation_side": "right",
43
  "truncation_strategy": "longest_first",
44
  "unk_token": "<unk>",
45
  "use_default_system_prompt": false
 
39
  "spaces_between_special_tokens": false,
40
  "stride": 0,
41
  "tokenizer_class": "LlamaTokenizer",
42
+ "truncation_side": "left",
43
  "truncation_strategy": "longest_first",
44
  "unk_token": "<unk>",
45
  "use_default_system_prompt": false