Hugging Face's logo

pabRomero
/
BART-Simplification-Paragraph-ONNX

ONNX
bart
Model card Files Community
pabRomero commited on
Commit
ffbf960
·
verified ·
1 Parent(s): 95d537e

Uploaded ONNX files

Browse files
Files changed (41) hide show
  1. added_tokens.json +100 -0
  2. config.json +74 -0
  3. generation_config.json +9 -0
  4. merges.txt +0 -0
  5. onnx/decoder_model.onnx +3 -0
  6. onnx/decoder_model_bnb4.onnx +3 -0
  7. onnx/decoder_model_fp16.onnx +3 -0
  8. onnx/decoder_model_int8.onnx +3 -0
  9. onnx/decoder_model_merged.onnx +3 -0
  10. onnx/decoder_model_merged_bnb4.onnx +3 -0
  11. onnx/decoder_model_merged_fp16.onnx +3 -0
  12. onnx/decoder_model_merged_int8.onnx +3 -0
  13. onnx/decoder_model_merged_q4.onnx +3 -0
  14. onnx/decoder_model_merged_q4f16.onnx +3 -0
  15. onnx/decoder_model_merged_quantized.onnx +3 -0
  16. onnx/decoder_model_merged_uint8.onnx +3 -0
  17. onnx/decoder_model_q4.onnx +3 -0
  18. onnx/decoder_model_q4f16.onnx +3 -0
  19. onnx/decoder_model_quantized.onnx +3 -0
  20. onnx/decoder_model_uint8.onnx +3 -0
  21. onnx/decoder_with_past_model.onnx +3 -0
  22. onnx/decoder_with_past_model_bnb4.onnx +3 -0
  23. onnx/decoder_with_past_model_fp16.onnx +3 -0
  24. onnx/decoder_with_past_model_int8.onnx +3 -0
  25. onnx/decoder_with_past_model_q4.onnx +3 -0
  26. onnx/decoder_with_past_model_q4f16.onnx +3 -0
  27. onnx/decoder_with_past_model_quantized.onnx +3 -0
  28. onnx/decoder_with_past_model_uint8.onnx +3 -0
  29. onnx/encoder_model.onnx +3 -0
  30. onnx/encoder_model_bnb4.onnx +3 -0
  31. onnx/encoder_model_fp16.onnx +3 -0
  32. onnx/encoder_model_int8.onnx +3 -0
  33. onnx/encoder_model_q4.onnx +3 -0
  34. onnx/encoder_model_q4f16.onnx +3 -0
  35. onnx/encoder_model_quantized.onnx +3 -0
  36. onnx/encoder_model_uint8.onnx +3 -0
  37. quantize_config.json +18 -0
  38. special_tokens_map.json +51 -0
  39. tokenizer.json +0 -0
  40. tokenizer_config.json +846 -0
  41. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<DEPENDENCYTREEDEPTHRATIO_0.25>": 50266,
3
+ "<DEPENDENCYTREEDEPTHRATIO_0.2>": 50265,
4
+ "<DEPENDENCYTREEDEPTHRATIO_0.35>": 50268,
5
+ "<DEPENDENCYTREEDEPTHRATIO_0.3>": 50267,
6
+ "<DEPENDENCYTREEDEPTHRATIO_0.45>": 50270,
7
+ "<DEPENDENCYTREEDEPTHRATIO_0.4>": 50269,
8
+ "<DEPENDENCYTREEDEPTHRATIO_0.55>": 50272,
9
+ "<DEPENDENCYTREEDEPTHRATIO_0.5>": 50271,
10
+ "<DEPENDENCYTREEDEPTHRATIO_0.65>": 50274,
11
+ "<DEPENDENCYTREEDEPTHRATIO_0.6>": 50273,
12
+ "<DEPENDENCYTREEDEPTHRATIO_0.75>": 50276,
13
+ "<DEPENDENCYTREEDEPTHRATIO_0.7>": 50275,
14
+ "<DEPENDENCYTREEDEPTHRATIO_0.85>": 50278,
15
+ "<DEPENDENCYTREEDEPTHRATIO_0.8>": 50277,
16
+ "<DEPENDENCYTREEDEPTHRATIO_0.95>": 50280,
17
+ "<DEPENDENCYTREEDEPTHRATIO_0.9>": 50279,
18
+ "<DEPENDENCYTREEDEPTHRATIO_1.05>": 50282,
19
+ "<DEPENDENCYTREEDEPTHRATIO_1.0>": 50281,
20
+ "<DEPENDENCYTREEDEPTHRATIO_1.15>": 50284,
21
+ "<DEPENDENCYTREEDEPTHRATIO_1.1>": 50283,
22
+ "<DEPENDENCYTREEDEPTHRATIO_1.25>": 50286,
23
+ "<DEPENDENCYTREEDEPTHRATIO_1.2>": 50285,
24
+ "<DEPENDENCYTREEDEPTHRATIO_1.35>": 50288,
25
+ "<DEPENDENCYTREEDEPTHRATIO_1.3>": 50287,
26
+ "<DEPENDENCYTREEDEPTHRATIO_1.45>": 50290,
27
+ "<DEPENDENCYTREEDEPTHRATIO_1.4>": 50289,
28
+ "<DEPENDENCYTREEDEPTHRATIO_1.5>": 50291,
29
+ "<LENGTHRATIO_0.25>": 50337,
30
+ "<LENGTHRATIO_0.2>": 50336,
31
+ "<LENGTHRATIO_0.35>": 50339,
32
+ "<LENGTHRATIO_0.3>": 50338,
33
+ "<LENGTHRATIO_0.45>": 50341,
34
+ "<LENGTHRATIO_0.4>": 50340,
35
+ "<LENGTHRATIO_0.55>": 50343,
36
+ "<LENGTHRATIO_0.5>": 50342,
37
+ "<LENGTHRATIO_0.65>": 50345,
38
+ "<LENGTHRATIO_0.6>": 50344,
39
+ "<LENGTHRATIO_0.75>": 50347,
40
+ "<LENGTHRATIO_0.7>": 50346,
41
+ "<LENGTHRATIO_0.85>": 50349,
42
+ "<LENGTHRATIO_0.8>": 50348,
43
+ "<LENGTHRATIO_0.95>": 50351,
44
+ "<LENGTHRATIO_0.9>": 50350,
45
+ "<LENGTHRATIO_1.05>": 50353,
46
+ "<LENGTHRATIO_1.0>": 50352,
47
+ "<LENGTHRATIO_1.15>": 50355,
48
+ "<LENGTHRATIO_1.1>": 50354,
49
+ "<LENGTHRATIO_1.25>": 50357,
50
+ "<LENGTHRATIO_1.2>": 50356,
51
+ "<LENGTHRATIO_1.35>": 50359,
52
+ "<LENGTHRATIO_1.3>": 50358,
53
+ "<LENGTHRATIO_1.45>": 50361,
54
+ "<LENGTHRATIO_1.4>": 50360,
55
+ "<LENGTHRATIO_1.5>": 50362,
56
+ "<REPLACEONLYLEVENSHTEIN_0.25>": 50320,
57
+ "<REPLACEONLYLEVENSHTEIN_0.2>": 50319,
58
+ "<REPLACEONLYLEVENSHTEIN_0.35>": 50322,
59
+ "<REPLACEONLYLEVENSHTEIN_0.3>": 50321,
60
+ "<REPLACEONLYLEVENSHTEIN_0.45>": 50324,
61
+ "<REPLACEONLYLEVENSHTEIN_0.4>": 50323,
62
+ "<REPLACEONLYLEVENSHTEIN_0.55>": 50326,
63
+ "<REPLACEONLYLEVENSHTEIN_0.5>": 50325,
64
+ "<REPLACEONLYLEVENSHTEIN_0.65>": 50328,
65
+ "<REPLACEONLYLEVENSHTEIN_0.6>": 50327,
66
+ "<REPLACEONLYLEVENSHTEIN_0.75>": 50330,
67
+ "<REPLACEONLYLEVENSHTEIN_0.7>": 50329,
68
+ "<REPLACEONLYLEVENSHTEIN_0.85>": 50332,
69
+ "<REPLACEONLYLEVENSHTEIN_0.8>": 50331,
70
+ "<REPLACEONLYLEVENSHTEIN_0.95>": 50334,
71
+ "<REPLACEONLYLEVENSHTEIN_0.9>": 50333,
72
+ "<REPLACEONLYLEVENSHTEIN_1.0>": 50335,
73
+ "<WORDRANKRATIO_0.25>": 50293,
74
+ "<WORDRANKRATIO_0.2>": 50292,
75
+ "<WORDRANKRATIO_0.35>": 50295,
76
+ "<WORDRANKRATIO_0.3>": 50294,
77
+ "<WORDRANKRATIO_0.45>": 50297,
78
+ "<WORDRANKRATIO_0.4>": 50296,
79
+ "<WORDRANKRATIO_0.55>": 50299,
80
+ "<WORDRANKRATIO_0.5>": 50298,
81
+ "<WORDRANKRATIO_0.65>": 50301,
82
+ "<WORDRANKRATIO_0.6>": 50300,
83
+ "<WORDRANKRATIO_0.75>": 50303,
84
+ "<WORDRANKRATIO_0.7>": 50302,
85
+ "<WORDRANKRATIO_0.85>": 50305,
86
+ "<WORDRANKRATIO_0.8>": 50304,
87
+ "<WORDRANKRATIO_0.95>": 50307,
88
+ "<WORDRANKRATIO_0.9>": 50306,
89
+ "<WORDRANKRATIO_1.05>": 50309,
90
+ "<WORDRANKRATIO_1.0>": 50308,
91
+ "<WORDRANKRATIO_1.15>": 50311,
92
+ "<WORDRANKRATIO_1.1>": 50310,
93
+ "<WORDRANKRATIO_1.25>": 50313,
94
+ "<WORDRANKRATIO_1.2>": 50312,
95
+ "<WORDRANKRATIO_1.35>": 50315,
96
+ "<WORDRANKRATIO_1.3>": 50314,
97
+ "<WORDRANKRATIO_1.45>": 50317,
98
+ "<WORDRANKRATIO_1.4>": 50316,
99
+ "<WORDRANKRATIO_1.5>": 50318
100
+ }
config.json ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_attn_implementation_autoset": true,
3
+ "_name_or_path": "pabRomero/BART-Simplification-Paragraph",
4
+ "activation_dropout": 0.1,
5
+ "activation_function": "gelu",
6
+ "add_bias_logits": false,
7
+ "add_final_layer_norm": false,
8
+ "architectures": [
9
+ "BartForConditionalGeneration"
10
+ ],
11
+ "attention_dropout": 0.1,
12
+ "bos_token_id": 0,
13
+ "classif_dropout": 0.1,
14
+ "classifier_dropout": 0.0,
15
+ "d_model": 768,
16
+ "decoder_attention_heads": 12,
17
+ "decoder_ffn_dim": 3072,
18
+ "decoder_layerdrop": 0.0,
19
+ "decoder_layers": 6,
20
+ "decoder_start_token_id": 2,
21
+ "dropout": 0.1,
22
+ "early_stopping": null,
23
+ "encoder_attention_heads": 12,
24
+ "encoder_ffn_dim": 3072,
25
+ "encoder_layerdrop": 0.0,
26
+ "encoder_layers": 6,
27
+ "eos_token_id": 2,
28
+ "forced_eos_token_id": 2,
29
+ "gradient_checkpointing": false,
30
+ "id2label": {
31
+ "0": "LABEL_0",
32
+ "1": "LABEL_1",
33
+ "2": "LABEL_2"
34
+ },
35
+ "init_std": 0.02,
36
+ "is_encoder_decoder": true,
37
+ "label2id": {
38
+ "LABEL_0": 0,
39
+ "LABEL_1": 1,
40
+ "LABEL_2": 2
41
+ },
42
+ "max_position_embeddings": 1024,
43
+ "model_type": "bart",
44
+ "no_repeat_ngram_size": null,
45
+ "normalize_before": false,
46
+ "normalize_embedding": true,
47
+ "num_beams": null,
48
+ "num_hidden_layers": 6,
49
+ "pad_token_id": 1,
50
+ "scale_embedding": false,
51
+ "task_specific_params": {
52
+ "summarization": {
53
+ "length_penalty": 1.0,
54
+ "max_length": 128,
55
+ "min_length": 12,
56
+ "num_beams": 4
57
+ },
58
+ "summarization_cnn": {
59
+ "length_penalty": 2.0,
60
+ "max_length": 142,
61
+ "min_length": 56,
62
+ "num_beams": 4
63
+ },
64
+ "summarization_xsum": {
65
+ "length_penalty": 1.0,
66
+ "max_length": 62,
67
+ "min_length": 11,
68
+ "num_beams": 6
69
+ }
70
+ },
71
+ "transformers_version": "4.46.3",
72
+ "use_cache": true,
73
+ "vocab_size": 50363
74
+ }
generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "decoder_start_token_id": 2,
5
+ "eos_token_id": 2,
6
+ "forced_eos_token_id": 2,
7
+ "pad_token_id": 1,
8
+ "transformers_version": "4.46.3"
9
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
onnx/decoder_model.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7932b55753e951026804fe22652be2f6f3d989d4b87fbe4c74da4825bc28153
3
+ size 539704944
onnx/decoder_model_bnb4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1681f7d98a6df1248aa50b693f03c68d467a252dfd65809037c91e4a1316f9d
3
+ size 212114196
onnx/decoder_model_fp16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1f5e143f591fec002bfe48f244455a28e3364c0f7192fd2a22dab8c629ddec2c
3
+ size 269991460
onnx/decoder_model_int8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:621a0f2177784cf65bb0f2620f475c7eb7a576cc159bae2b084f2bc3568267c8
3
+ size 136025149
onnx/decoder_model_merged.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53a90a782c67f0b136f92749f25f4cbbfadebea962581130e1108e0f363c3586
3
+ size 540236740
onnx/decoder_model_merged_bnb4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0031afa607f344e671073db1123bb7b5fd85df5e3fc6d7d891907eccd760c353
3
+ size 212653166
onnx/decoder_model_merged_fp16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41dc2d006249833a53299b582177a195af5db03792c1e645a709c7369a98e159
3
+ size 526079842
onnx/decoder_model_merged_int8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85a1e3116b252d9fd893b4d59e4d8ace1cd5232c9725293a821fc7541249057e
3
+ size 136618566
onnx/decoder_model_merged_q4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa9b4730d169acdc40675d5ba1f7b1eb2c622ef86d2929cbc27b05efb173da9f
3
+ size 218608653
onnx/decoder_model_merged_q4f16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7fcfa7594dff18c7635d6e048692d95372bc5c1233ff7295f68cb2fe34be946e
3
+ size 262279831
onnx/decoder_model_merged_quantized.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:85a1e3116b252d9fd893b4d59e4d8ace1cd5232c9725293a821fc7541249057e
3
+ size 136618566
onnx/decoder_model_merged_uint8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a3caf95556bd7e7ff3ef8c29307e66eb3c17fb5c3d4094c008d38a723caa2399
3
+ size 136618566
onnx/decoder_model_q4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24c7e44b6477c3180d5d458969c4afc52f7347c20e78dd4e1a6c79935c57c48f
3
+ size 218070124
onnx/decoder_model_q4f16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f63a4b323c31d91368ebaaea88d5308e63ed6dd626ea0a452415e139cd6e69f
3
+ size 133004144
onnx/decoder_model_quantized.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:621a0f2177784cf65bb0f2620f475c7eb7a576cc159bae2b084f2bc3568267c8
3
+ size 136025149
onnx/decoder_model_uint8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:555fbbe1985cc3c5de95b347cbfd20ecfcb6ca4ed355fac392c08116af5e6b10
3
+ size 136025149
onnx/decoder_with_past_model.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0b5fac8a7d9d11fafd8f7beeb6b361f3323e33f640019b06bd9ac4f85b30d0d
3
+ size 511288992
onnx/decoder_with_past_model_bnb4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a1cf558b60253929623edcc4f69f78f4f1c1d88438e3360ee17cb741b6537251
3
+ size 208026624
onnx/decoder_with_past_model_fp16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50859dd54ce92d2e08bd58cc44a5ee46485275e5f20b21ab0e8d0c2c64f031af
3
+ size 255752543
onnx/decoder_with_past_model_int8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04f7a0f09a35065423aebcb4a38dd3d3129afc2ac3e5029530e2f866addc6115
3
+ size 128784856
onnx/decoder_with_past_model_q4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f0bdb5b4b828c589d770df555f53f6718850eb181a0efb771a80879325f7156
3
+ size 213540268
onnx/decoder_with_past_model_q4f16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b8af6f7ee8924262cfe2fee66852215199a345222071e572623cc3e64854e73
3
+ size 128937892
onnx/decoder_with_past_model_quantized.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04f7a0f09a35065423aebcb4a38dd3d3129afc2ac3e5029530e2f866addc6115
3
+ size 128784856
onnx/decoder_with_past_model_uint8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e75d9c20a3aa4bbd548684c0c586e449660853119dfe88127fb6ddd5c0f01083
3
+ size 128784856
onnx/encoder_model.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25e8a54e9028c3830903c49ede27090d8d737506e9b8405a8eb4e70421f597e4
3
+ size 328105321
onnx/encoder_model_bnb4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:279f1d5cc9660d9527c0997fe41312e2d9b9175fc6e334f6598b4552ce049697
3
+ size 182129503
onnx/encoder_model_fp16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92fd58dd0df0db5aa224cbeb5d93f8f8e37e5e19d78ea86c16b952c358cbfb21
3
+ size 164114297
onnx/encoder_model_int8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14868506b1351e7540a04a74812fef770894ef5f455e0c20fe302b3181e641fa
3
+ size 82547728
onnx/encoder_model_q4.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b98b675f7c955c80d92929586d173e7259c26d6517bcc2980a93dd8f61ebebc7
3
+ size 184783447
onnx/encoder_model_q4f16.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d932c00ede0064cc3236057af85ff360d3a79f94623f5c27ceb90bb20845de33
3
+ size 103072920
onnx/encoder_model_quantized.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14868506b1351e7540a04a74812fef770894ef5f455e0c20fe302b3181e641fa
3
+ size 82547728
onnx/encoder_model_uint8.onnx ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20ab1ebb0a4f880ac37d0458964e2dff07e7f53d537af22359a014fc872c5bf2
3
+ size 82547728
quantize_config.json ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "modes": [
3
+ "fp16",
4
+ "q8",
5
+ "int8",
6
+ "uint8",
7
+ "q4",
8
+ "q4f16",
9
+ "bnb4"
10
+ ],
11
+ "per_channel": true,
12
+ "reduce_range": true,
13
+ "block_size": null,
14
+ "is_symmetric": true,
15
+ "accuracy_level": null,
16
+ "quant_type": 1,
17
+ "op_block_list": null
18
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "cls_token": {
10
+ "content": "<s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "eos_token": {
17
+ "content": "</s>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "mask_token": {
24
+ "content": "<mask>",
25
+ "lstrip": true,
26
+ "normalized": true,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "pad_token": {
31
+ "content": "<pad>",
32
+ "lstrip": false,
33
+ "normalized": true,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ },
37
+ "sep_token": {
38
+ "content": "</s>",
39
+ "lstrip": false,
40
+ "normalized": true,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
+ "unk_token": {
45
+ "content": "<unk>",
46
+ "lstrip": false,
47
+ "normalized": true,
48
+ "rstrip": false,
49
+ "single_word": false
50
+ }
51
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,846 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<s>",
6
+ "lstrip": false,
7
+ "normalized": true,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<pad>",
14
+ "lstrip": false,
15
+ "normalized": true,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": true,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "50264": {
37
+ "content": "<mask>",
38
+ "lstrip": true,
39
+ "normalized": true,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "50265": {
45
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.2>",
46
+ "lstrip": false,
47
+ "normalized": true,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": false
51
+ },
52
+ "50266": {
53
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.25>",
54
+ "lstrip": false,
55
+ "normalized": true,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": false
59
+ },
60
+ "50267": {
61
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.3>",
62
+ "lstrip": false,
63
+ "normalized": true,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": false
67
+ },
68
+ "50268": {
69
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.35>",
70
+ "lstrip": false,
71
+ "normalized": true,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": false
75
+ },
76
+ "50269": {
77
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.4>",
78
+ "lstrip": false,
79
+ "normalized": true,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": false
83
+ },
84
+ "50270": {
85
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.45>",
86
+ "lstrip": false,
87
+ "normalized": true,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": false
91
+ },
92
+ "50271": {
93
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.5>",
94
+ "lstrip": false,
95
+ "normalized": true,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": false
99
+ },
100
+ "50272": {
101
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.55>",
102
+ "lstrip": false,
103
+ "normalized": true,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": false
107
+ },
108
+ "50273": {
109
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.6>",
110
+ "lstrip": false,
111
+ "normalized": true,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": false
115
+ },
116
+ "50274": {
117
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.65>",
118
+ "lstrip": false,
119
+ "normalized": true,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": false
123
+ },
124
+ "50275": {
125
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.7>",
126
+ "lstrip": false,
127
+ "normalized": true,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": false
131
+ },
132
+ "50276": {
133
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.75>",
134
+ "lstrip": false,
135
+ "normalized": true,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": false
139
+ },
140
+ "50277": {
141
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.8>",
142
+ "lstrip": false,
143
+ "normalized": true,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": false
147
+ },
148
+ "50278": {
149
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.85>",
150
+ "lstrip": false,
151
+ "normalized": true,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": false
155
+ },
156
+ "50279": {
157
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.9>",
158
+ "lstrip": false,
159
+ "normalized": true,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": false
163
+ },
164
+ "50280": {
165
+ "content": "<DEPENDENCYTREEDEPTHRATIO_0.95>",
166
+ "lstrip": false,
167
+ "normalized": true,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": false
171
+ },
172
+ "50281": {
173
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.0>",
174
+ "lstrip": false,
175
+ "normalized": true,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": false
179
+ },
180
+ "50282": {
181
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.05>",
182
+ "lstrip": false,
183
+ "normalized": true,
184
+ "rstrip": false,
185
+ "single_word": false,
186
+ "special": false
187
+ },
188
+ "50283": {
189
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.1>",
190
+ "lstrip": false,
191
+ "normalized": true,
192
+ "rstrip": false,
193
+ "single_word": false,
194
+ "special": false
195
+ },
196
+ "50284": {
197
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.15>",
198
+ "lstrip": false,
199
+ "normalized": true,
200
+ "rstrip": false,
201
+ "single_word": false,
202
+ "special": false
203
+ },
204
+ "50285": {
205
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.2>",
206
+ "lstrip": false,
207
+ "normalized": true,
208
+ "rstrip": false,
209
+ "single_word": false,
210
+ "special": false
211
+ },
212
+ "50286": {
213
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.25>",
214
+ "lstrip": false,
215
+ "normalized": true,
216
+ "rstrip": false,
217
+ "single_word": false,
218
+ "special": false
219
+ },
220
+ "50287": {
221
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.3>",
222
+ "lstrip": false,
223
+ "normalized": true,
224
+ "rstrip": false,
225
+ "single_word": false,
226
+ "special": false
227
+ },
228
+ "50288": {
229
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.35>",
230
+ "lstrip": false,
231
+ "normalized": true,
232
+ "rstrip": false,
233
+ "single_word": false,
234
+ "special": false
235
+ },
236
+ "50289": {
237
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.4>",
238
+ "lstrip": false,
239
+ "normalized": true,
240
+ "rstrip": false,
241
+ "single_word": false,
242
+ "special": false
243
+ },
244
+ "50290": {
245
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.45>",
246
+ "lstrip": false,
247
+ "normalized": true,
248
+ "rstrip": false,
249
+ "single_word": false,
250
+ "special": false
251
+ },
252
+ "50291": {
253
+ "content": "<DEPENDENCYTREEDEPTHRATIO_1.5>",
254
+ "lstrip": false,
255
+ "normalized": true,
256
+ "rstrip": false,
257
+ "single_word": false,
258
+ "special": false
259
+ },
260
+ "50292": {
261
+ "content": "<WORDRANKRATIO_0.2>",
262
+ "lstrip": false,
263
+ "normalized": true,
264
+ "rstrip": false,
265
+ "single_word": false,
266
+ "special": false
267
+ },
268
+ "50293": {
269
+ "content": "<WORDRANKRATIO_0.25>",
270
+ "lstrip": false,
271
+ "normalized": true,
272
+ "rstrip": false,
273
+ "single_word": false,
274
+ "special": false
275
+ },
276
+ "50294": {
277
+ "content": "<WORDRANKRATIO_0.3>",
278
+ "lstrip": false,
279
+ "normalized": true,
280
+ "rstrip": false,
281
+ "single_word": false,
282
+ "special": false
283
+ },
284
+ "50295": {
285
+ "content": "<WORDRANKRATIO_0.35>",
286
+ "lstrip": false,
287
+ "normalized": true,
288
+ "rstrip": false,
289
+ "single_word": false,
290
+ "special": false
291
+ },
292
+ "50296": {
293
+ "content": "<WORDRANKRATIO_0.4>",
294
+ "lstrip": false,
295
+ "normalized": true,
296
+ "rstrip": false,
297
+ "single_word": false,
298
+ "special": false
299
+ },
300
+ "50297": {
301
+ "content": "<WORDRANKRATIO_0.45>",
302
+ "lstrip": false,
303
+ "normalized": true,
304
+ "rstrip": false,
305
+ "single_word": false,
306
+ "special": false
307
+ },
308
+ "50298": {
309
+ "content": "<WORDRANKRATIO_0.5>",
310
+ "lstrip": false,
311
+ "normalized": true,
312
+ "rstrip": false,
313
+ "single_word": false,
314
+ "special": false
315
+ },
316
+ "50299": {
317
+ "content": "<WORDRANKRATIO_0.55>",
318
+ "lstrip": false,
319
+ "normalized": true,
320
+ "rstrip": false,
321
+ "single_word": false,
322
+ "special": false
323
+ },
324
+ "50300": {
325
+ "content": "<WORDRANKRATIO_0.6>",
326
+ "lstrip": false,
327
+ "normalized": true,
328
+ "rstrip": false,
329
+ "single_word": false,
330
+ "special": false
331
+ },
332
+ "50301": {
333
+ "content": "<WORDRANKRATIO_0.65>",
334
+ "lstrip": false,
335
+ "normalized": true,
336
+ "rstrip": false,
337
+ "single_word": false,
338
+ "special": false
339
+ },
340
+ "50302": {
341
+ "content": "<WORDRANKRATIO_0.7>",
342
+ "lstrip": false,
343
+ "normalized": true,
344
+ "rstrip": false,
345
+ "single_word": false,
346
+ "special": false
347
+ },
348
+ "50303": {
349
+ "content": "<WORDRANKRATIO_0.75>",
350
+ "lstrip": false,
351
+ "normalized": true,
352
+ "rstrip": false,
353
+ "single_word": false,
354
+ "special": false
355
+ },
356
+ "50304": {
357
+ "content": "<WORDRANKRATIO_0.8>",
358
+ "lstrip": false,
359
+ "normalized": true,
360
+ "rstrip": false,
361
+ "single_word": false,
362
+ "special": false
363
+ },
364
+ "50305": {
365
+ "content": "<WORDRANKRATIO_0.85>",
366
+ "lstrip": false,
367
+ "normalized": true,
368
+ "rstrip": false,
369
+ "single_word": false,
370
+ "special": false
371
+ },
372
+ "50306": {
373
+ "content": "<WORDRANKRATIO_0.9>",
374
+ "lstrip": false,
375
+ "normalized": true,
376
+ "rstrip": false,
377
+ "single_word": false,
378
+ "special": false
379
+ },
380
+ "50307": {
381
+ "content": "<WORDRANKRATIO_0.95>",
382
+ "lstrip": false,
383
+ "normalized": true,
384
+ "rstrip": false,
385
+ "single_word": false,
386
+ "special": false
387
+ },
388
+ "50308": {
389
+ "content": "<WORDRANKRATIO_1.0>",
390
+ "lstrip": false,
391
+ "normalized": true,
392
+ "rstrip": false,
393
+ "single_word": false,
394
+ "special": false
395
+ },
396
+ "50309": {
397
+ "content": "<WORDRANKRATIO_1.05>",
398
+ "lstrip": false,
399
+ "normalized": true,
400
+ "rstrip": false,
401
+ "single_word": false,
402
+ "special": false
403
+ },
404
+ "50310": {
405
+ "content": "<WORDRANKRATIO_1.1>",
406
+ "lstrip": false,
407
+ "normalized": true,
408
+ "rstrip": false,
409
+ "single_word": false,
410
+ "special": false
411
+ },
412
+ "50311": {
413
+ "content": "<WORDRANKRATIO_1.15>",
414
+ "lstrip": false,
415
+ "normalized": true,
416
+ "rstrip": false,
417
+ "single_word": false,
418
+ "special": false
419
+ },
420
+ "50312": {
421
+ "content": "<WORDRANKRATIO_1.2>",
422
+ "lstrip": false,
423
+ "normalized": true,
424
+ "rstrip": false,
425
+ "single_word": false,
426
+ "special": false
427
+ },
428
+ "50313": {
429
+ "content": "<WORDRANKRATIO_1.25>",
430
+ "lstrip": false,
431
+ "normalized": true,
432
+ "rstrip": false,
433
+ "single_word": false,
434
+ "special": false
435
+ },
436
+ "50314": {
437
+ "content": "<WORDRANKRATIO_1.3>",
438
+ "lstrip": false,
439
+ "normalized": true,
440
+ "rstrip": false,
441
+ "single_word": false,
442
+ "special": false
443
+ },
444
+ "50315": {
445
+ "content": "<WORDRANKRATIO_1.35>",
446
+ "lstrip": false,
447
+ "normalized": true,
448
+ "rstrip": false,
449
+ "single_word": false,
450
+ "special": false
451
+ },
452
+ "50316": {
453
+ "content": "<WORDRANKRATIO_1.4>",
454
+ "lstrip": false,
455
+ "normalized": true,
456
+ "rstrip": false,
457
+ "single_word": false,
458
+ "special": false
459
+ },
460
+ "50317": {
461
+ "content": "<WORDRANKRATIO_1.45>",
462
+ "lstrip": false,
463
+ "normalized": true,
464
+ "rstrip": false,
465
+ "single_word": false,
466
+ "special": false
467
+ },
468
+ "50318": {
469
+ "content": "<WORDRANKRATIO_1.5>",
470
+ "lstrip": false,
471
+ "normalized": true,
472
+ "rstrip": false,
473
+ "single_word": false,
474
+ "special": false
475
+ },
476
+ "50319": {
477
+ "content": "<REPLACEONLYLEVENSHTEIN_0.2>",
478
+ "lstrip": false,
479
+ "normalized": true,
480
+ "rstrip": false,
481
+ "single_word": false,
482
+ "special": false
483
+ },
484
+ "50320": {
485
+ "content": "<REPLACEONLYLEVENSHTEIN_0.25>",
486
+ "lstrip": false,
487
+ "normalized": true,
488
+ "rstrip": false,
489
+ "single_word": false,
490
+ "special": false
491
+ },
492
+ "50321": {
493
+ "content": "<REPLACEONLYLEVENSHTEIN_0.3>",
494
+ "lstrip": false,
495
+ "normalized": true,
496
+ "rstrip": false,
497
+ "single_word": false,
498
+ "special": false
499
+ },
500
+ "50322": {
501
+ "content": "<REPLACEONLYLEVENSHTEIN_0.35>",
502
+ "lstrip": false,
503
+ "normalized": true,
504
+ "rstrip": false,
505
+ "single_word": false,
506
+ "special": false
507
+ },
508
+ "50323": {
509
+ "content": "<REPLACEONLYLEVENSHTEIN_0.4>",
510
+ "lstrip": false,
511
+ "normalized": true,
512
+ "rstrip": false,
513
+ "single_word": false,
514
+ "special": false
515
+ },
516
+ "50324": {
517
+ "content": "<REPLACEONLYLEVENSHTEIN_0.45>",
518
+ "lstrip": false,
519
+ "normalized": true,
520
+ "rstrip": false,
521
+ "single_word": false,
522
+ "special": false
523
+ },
524
+ "50325": {
525
+ "content": "<REPLACEONLYLEVENSHTEIN_0.5>",
526
+ "lstrip": false,
527
+ "normalized": true,
528
+ "rstrip": false,
529
+ "single_word": false,
530
+ "special": false
531
+ },
532
+ "50326": {
533
+ "content": "<REPLACEONLYLEVENSHTEIN_0.55>",
534
+ "lstrip": false,
535
+ "normalized": true,
536
+ "rstrip": false,
537
+ "single_word": false,
538
+ "special": false
539
+ },
540
+ "50327": {
541
+ "content": "<REPLACEONLYLEVENSHTEIN_0.6>",
542
+ "lstrip": false,
543
+ "normalized": true,
544
+ "rstrip": false,
545
+ "single_word": false,
546
+ "special": false
547
+ },
548
+ "50328": {
549
+ "content": "<REPLACEONLYLEVENSHTEIN_0.65>",
550
+ "lstrip": false,
551
+ "normalized": true,
552
+ "rstrip": false,
553
+ "single_word": false,
554
+ "special": false
555
+ },
556
+ "50329": {
557
+ "content": "<REPLACEONLYLEVENSHTEIN_0.7>",
558
+ "lstrip": false,
559
+ "normalized": true,
560
+ "rstrip": false,
561
+ "single_word": false,
562
+ "special": false
563
+ },
564
+ "50330": {
565
+ "content": "<REPLACEONLYLEVENSHTEIN_0.75>",
566
+ "lstrip": false,
567
+ "normalized": true,
568
+ "rstrip": false,
569
+ "single_word": false,
570
+ "special": false
571
+ },
572
+ "50331": {
573
+ "content": "<REPLACEONLYLEVENSHTEIN_0.8>",
574
+ "lstrip": false,
575
+ "normalized": true,
576
+ "rstrip": false,
577
+ "single_word": false,
578
+ "special": false
579
+ },
580
+ "50332": {
581
+ "content": "<REPLACEONLYLEVENSHTEIN_0.85>",
582
+ "lstrip": false,
583
+ "normalized": true,
584
+ "rstrip": false,
585
+ "single_word": false,
586
+ "special": false
587
+ },
588
+ "50333": {
589
+ "content": "<REPLACEONLYLEVENSHTEIN_0.9>",
590
+ "lstrip": false,
591
+ "normalized": true,
592
+ "rstrip": false,
593
+ "single_word": false,
594
+ "special": false
595
+ },
596
+ "50334": {
597
+ "content": "<REPLACEONLYLEVENSHTEIN_0.95>",
598
+ "lstrip": false,
599
+ "normalized": true,
600
+ "rstrip": false,
601
+ "single_word": false,
602
+ "special": false
603
+ },
604
+ "50335": {
605
+ "content": "<REPLACEONLYLEVENSHTEIN_1.0>",
606
+ "lstrip": false,
607
+ "normalized": true,
608
+ "rstrip": false,
609
+ "single_word": false,
610
+ "special": false
611
+ },
612
+ "50336": {
613
+ "content": "<LENGTHRATIO_0.2>",
614
+ "lstrip": false,
615
+ "normalized": true,
616
+ "rstrip": false,
617
+ "single_word": false,
618
+ "special": false
619
+ },
620
+ "50337": {
621
+ "content": "<LENGTHRATIO_0.25>",
622
+ "lstrip": false,
623
+ "normalized": true,
624
+ "rstrip": false,
625
+ "single_word": false,
626
+ "special": false
627
+ },
628
+ "50338": {
629
+ "content": "<LENGTHRATIO_0.3>",
630
+ "lstrip": false,
631
+ "normalized": true,
632
+ "rstrip": false,
633
+ "single_word": false,
634
+ "special": false
635
+ },
636
+ "50339": {
637
+ "content": "<LENGTHRATIO_0.35>",
638
+ "lstrip": false,
639
+ "normalized": true,
640
+ "rstrip": false,
641
+ "single_word": false,
642
+ "special": false
643
+ },
644
+ "50340": {
645
+ "content": "<LENGTHRATIO_0.4>",
646
+ "lstrip": false,
647
+ "normalized": true,
648
+ "rstrip": false,
649
+ "single_word": false,
650
+ "special": false
651
+ },
652
+ "50341": {
653
+ "content": "<LENGTHRATIO_0.45>",
654
+ "lstrip": false,
655
+ "normalized": true,
656
+ "rstrip": false,
657
+ "single_word": false,
658
+ "special": false
659
+ },
660
+ "50342": {
661
+ "content": "<LENGTHRATIO_0.5>",
662
+ "lstrip": false,
663
+ "normalized": true,
664
+ "rstrip": false,
665
+ "single_word": false,
666
+ "special": false
667
+ },
668
+ "50343": {
669
+ "content": "<LENGTHRATIO_0.55>",
670
+ "lstrip": false,
671
+ "normalized": true,
672
+ "rstrip": false,
673
+ "single_word": false,
674
+ "special": false
675
+ },
676
+ "50344": {
677
+ "content": "<LENGTHRATIO_0.6>",
678
+ "lstrip": false,
679
+ "normalized": true,
680
+ "rstrip": false,
681
+ "single_word": false,
682
+ "special": false
683
+ },
684
+ "50345": {
685
+ "content": "<LENGTHRATIO_0.65>",
686
+ "lstrip": false,
687
+ "normalized": true,
688
+ "rstrip": false,
689
+ "single_word": false,
690
+ "special": false
691
+ },
692
+ "50346": {
693
+ "content": "<LENGTHRATIO_0.7>",
694
+ "lstrip": false,
695
+ "normalized": true,
696
+ "rstrip": false,
697
+ "single_word": false,
698
+ "special": false
699
+ },
700
+ "50347": {
701
+ "content": "<LENGTHRATIO_0.75>",
702
+ "lstrip": false,
703
+ "normalized": true,
704
+ "rstrip": false,
705
+ "single_word": false,
706
+ "special": false
707
+ },
708
+ "50348": {
709
+ "content": "<LENGTHRATIO_0.8>",
710
+ "lstrip": false,
711
+ "normalized": true,
712
+ "rstrip": false,
713
+ "single_word": false,
714
+ "special": false
715
+ },
716
+ "50349": {
717
+ "content": "<LENGTHRATIO_0.85>",
718
+ "lstrip": false,
719
+ "normalized": true,
720
+ "rstrip": false,
721
+ "single_word": false,
722
+ "special": false
723
+ },
724
+ "50350": {
725
+ "content": "<LENGTHRATIO_0.9>",
726
+ "lstrip": false,
727
+ "normalized": true,
728
+ "rstrip": false,
729
+ "single_word": false,
730
+ "special": false
731
+ },
732
+ "50351": {
733
+ "content": "<LENGTHRATIO_0.95>",
734
+ "lstrip": false,
735
+ "normalized": true,
736
+ "rstrip": false,
737
+ "single_word": false,
738
+ "special": false
739
+ },
740
+ "50352": {
741
+ "content": "<LENGTHRATIO_1.0>",
742
+ "lstrip": false,
743
+ "normalized": true,
744
+ "rstrip": false,
745
+ "single_word": false,
746
+ "special": false
747
+ },
748
+ "50353": {
749
+ "content": "<LENGTHRATIO_1.05>",
750
+ "lstrip": false,
751
+ "normalized": true,
752
+ "rstrip": false,
753
+ "single_word": false,
754
+ "special": false
755
+ },
756
+ "50354": {
757
+ "content": "<LENGTHRATIO_1.1>",
758
+ "lstrip": false,
759
+ "normalized": true,
760
+ "rstrip": false,
761
+ "single_word": false,
762
+ "special": false
763
+ },
764
+ "50355": {
765
+ "content": "<LENGTHRATIO_1.15>",
766
+ "lstrip": false,
767
+ "normalized": true,
768
+ "rstrip": false,
769
+ "single_word": false,
770
+ "special": false
771
+ },
772
+ "50356": {
773
+ "content": "<LENGTHRATIO_1.2>",
774
+ "lstrip": false,
775
+ "normalized": true,
776
+ "rstrip": false,
777
+ "single_word": false,
778
+ "special": false
779
+ },
780
+ "50357": {
781
+ "content": "<LENGTHRATIO_1.25>",
782
+ "lstrip": false,
783
+ "normalized": true,
784
+ "rstrip": false,
785
+ "single_word": false,
786
+ "special": false
787
+ },
788
+ "50358": {
789
+ "content": "<LENGTHRATIO_1.3>",
790
+ "lstrip": false,
791
+ "normalized": true,
792
+ "rstrip": false,
793
+ "single_word": false,
794
+ "special": false
795
+ },
796
+ "50359": {
797
+ "content": "<LENGTHRATIO_1.35>",
798
+ "lstrip": false,
799
+ "normalized": true,
800
+ "rstrip": false,
801
+ "single_word": false,
802
+ "special": false
803
+ },
804
+ "50360": {
805
+ "content": "<LENGTHRATIO_1.4>",
806
+ "lstrip": false,
807
+ "normalized": true,
808
+ "rstrip": false,
809
+ "single_word": false,
810
+ "special": false
811
+ },
812
+ "50361": {
813
+ "content": "<LENGTHRATIO_1.45>",
814
+ "lstrip": false,
815
+ "normalized": true,
816
+ "rstrip": false,
817
+ "single_word": false,
818
+ "special": false
819
+ },
820
+ "50362": {
821
+ "content": "<LENGTHRATIO_1.5>",
822
+ "lstrip": false,
823
+ "normalized": true,
824
+ "rstrip": false,
825
+ "single_word": false,
826
+ "special": false
827
+ }
828
+ },
829
+ "bos_token": "<s>",
830
+ "clean_up_tokenization_spaces": false,
831
+ "cls_token": "<s>",
832
+ "eos_token": "</s>",
833
+ "errors": "replace",
834
+ "extra_special_tokens": {},
835
+ "mask_token": "<mask>",
836
+ "max_length": 768,
837
+ "model_max_length": 1000000000000000019884624838656,
838
+ "pad_token": "<pad>",
839
+ "sep_token": "</s>",
840
+ "stride": 0,
841
+ "tokenizer_class": "BartTokenizer",
842
+ "trim_offsets": true,
843
+ "truncation_side": "right",
844
+ "truncation_strategy": "longest_first",
845
+ "unk_token": "<unk>"
846
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff