Hugging Face's logo

ychenNLP
/
nllb-200-distilled-1.3B-easyproject

Translation
Transformers
PyTorch
m2m_100
text2text-generation
nllb
Model card Files Community
4
edchengg commited on
Commit
0602737
•
1 Parent(s): 9d1cf4e

Huggingface nllb tokenization issue

Browse files
Files changed (22) hide show
  1. .gitattributes +0 -34
  2. config copy.json +0 -34
  3. config.json +0 -34
  4. generation_config.json +0 -9
  5. optimizer copy.pt +0 -3
  6. optimizer.pt +0 -3
  7. pytorch_model copy.bin +0 -3
  8. pytorch_model.bin +0 -3
  9. rng_state copy.pth +0 -3
  10. rng_state.pth +0 -3
  11. scheduler copy.pt +0 -3
  12. scheduler.pt +0 -3
  13. sentencepiece.bpe copy.model +0 -3
  14. sentencepiece.bpe.model +0 -3
  15. special_tokens_map copy.json +0 -219
  16. special_tokens_map.json +0 -219
  17. tokenizer_config copy.json +0 -25
  18. tokenizer_config.json +0 -25
  19. trainer_state copy.json +0 -15
  20. trainer_state.json +0 -15
  21. training_args copy.bin +0 -3
  22. training_args.bin +0 -3
.gitattributes DELETED
@@ -1,34 +0,0 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tflite filter=lfs diff=lfs merge=lfs -text
29
- *.tgz filter=lfs diff=lfs merge=lfs -text
30
- *.wasm filter=lfs diff=lfs merge=lfs -text
31
- *.xz filter=lfs diff=lfs merge=lfs -text
32
- *.zip filter=lfs diff=lfs merge=lfs -text
33
- *.zst filter=lfs diff=lfs merge=lfs -text
34
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
config copy.json DELETED
@@ -1,34 +0,0 @@
1
- {
2
- "_name_or_path": "facebook/nllb-200-distilled-1.3B",
3
- "activation_dropout": 0.0,
4
- "activation_function": "relu",
5
- "architectures": [
6
- "M2M100ForConditionalGeneration"
7
- ],
8
- "attention_dropout": 0.1,
9
- "bos_token_id": 0,
10
- "d_model": 1024,
11
- "decoder_attention_heads": 16,
12
- "decoder_ffn_dim": 8192,
13
- "decoder_layerdrop": 0,
14
- "decoder_layers": 24,
15
- "decoder_start_token_id": 2,
16
- "dropout": 0.1,
17
- "encoder_attention_heads": 16,
18
- "encoder_ffn_dim": 8192,
19
- "encoder_layerdrop": 0,
20
- "encoder_layers": 24,
21
- "eos_token_id": 2,
22
- "init_std": 0.02,
23
- "is_encoder_decoder": true,
24
- "max_length": 200,
25
- "max_position_embeddings": 1024,
26
- "model_type": "m2m_100",
27
- "num_hidden_layers": 24,
28
- "pad_token_id": 1,
29
- "scale_embedding": true,
30
- "torch_dtype": "float32",
31
- "transformers_version": "4.29.2",
32
- "use_cache": true,
33
- "vocab_size": 256206
34
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
config.json DELETED
@@ -1,34 +0,0 @@
1
- {
2
- "_name_or_path": "facebook/nllb-200-distilled-1.3B",
3
- "activation_dropout": 0.0,
4
- "activation_function": "relu",
5
- "architectures": [
6
- "M2M100ForConditionalGeneration"
7
- ],
8
- "attention_dropout": 0.1,
9
- "bos_token_id": 0,
10
- "d_model": 1024,
11
- "decoder_attention_heads": 16,
12
- "decoder_ffn_dim": 8192,
13
- "decoder_layerdrop": 0,
14
- "decoder_layers": 24,
15
- "decoder_start_token_id": 2,
16
- "dropout": 0.1,
17
- "encoder_attention_heads": 16,
18
- "encoder_ffn_dim": 8192,
19
- "encoder_layerdrop": 0,
20
- "encoder_layers": 24,
21
- "eos_token_id": 2,
22
- "init_std": 0.02,
23
- "is_encoder_decoder": true,
24
- "max_length": 200,
25
- "max_position_embeddings": 1024,
26
- "model_type": "m2m_100",
27
- "num_hidden_layers": 24,
28
- "pad_token_id": 1,
29
- "scale_embedding": true,
30
- "torch_dtype": "float32",
31
- "transformers_version": "4.25.1",
32
- "use_cache": true,
33
- "vocab_size": 256206
34
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
generation_config.json DELETED
@@ -1,9 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 0,
4
- "decoder_start_token_id": 2,
5
- "eos_token_id": 2,
6
- "max_length": 200,
7
- "pad_token_id": 1,
8
- "transformers_version": "4.29.2"
9
- }
 
 
 
 
 
 
 
 
 
 
optimizer copy.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ddc1e1a5aabcf48363cf287a46d3ff983a5dc64d51acd2e6e4d8b2f2632494f1
3
- size 369450217
 
 
 
 
optimizer.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a1ed769360ea3182400197c682c290d3810615b5f15f7920141150bb51798c5a
3
- size 369452969
 
 
 
 
pytorch_model copy.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:44c94e81263bb6201728a78a19256d9a681f56bd65bc8e2bc62efc83889eb6ea
3
- size 5491302146
 
 
 
 
pytorch_model.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7d1d2c85d3c326806f8c5733fa9c26ea0b602e879f1e1e1c428792e1858848cd
3
- size 5491302146
 
 
 
 
rng_state copy.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:6facffd74bec5258e8fd79ea06d0ee8b0ef4398f5eb3a89f6e32a315245a08d3
3
- size 14503
 
 
 
 
rng_state.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ccef3a832c0a43e0982f995223966f29558207dc8933a3a2ac44e3070d2db83c
3
- size 14503
 
 
 
 
scheduler copy.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ed70e833c6c2092be3980743d06828aaf3cf489390300379089a83c5b56e2ebc
3
- size 623
 
 
 
 
scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:dba8231a7061a6ded3dce8a6bb8e6f792c2b37751a0e1b626cd3e7eedf685418
3
- size 623
 
 
 
 
sentencepiece.bpe copy.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a
3
- size 4852054
 
 
 
 
sentencepiece.bpe.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:14bb8dfb35c0ffdea7bc01e56cea38b9e3d5efcdcb9c251d6b40538e1aab555a
3
- size 4852054
 
 
 
 
special_tokens_map copy.json DELETED
@@ -1,219 +0,0 @@
1
- {
2
- "additional_special_tokens": [
3
- "ace_Arab",
4
- "ace_Latn",
5
- "acm_Arab",
6
- "acq_Arab",
7
- "aeb_Arab",
8
- "afr_Latn",
9
- "ajp_Arab",
10
- "aka_Latn",
11
- "amh_Ethi",
12
- "apc_Arab",
13
- "arb_Arab",
14
- "ars_Arab",
15
- "ary_Arab",
16
- "arz_Arab",
17
- "asm_Beng",
18
- "ast_Latn",
19
- "awa_Deva",
20
- "ayr_Latn",
21
- "azb_Arab",
22
- "azj_Latn",
23
- "bak_Cyrl",
24
- "bam_Latn",
25
- "ban_Latn",
26
- "bel_Cyrl",
27
- "bem_Latn",
28
- "ben_Beng",
29
- "bho_Deva",
30
- "bjn_Arab",
31
- "bjn_Latn",
32
- "bod_Tibt",
33
- "bos_Latn",
34
- "bug_Latn",
35
- "bul_Cyrl",
36
- "cat_Latn",
37
- "ceb_Latn",
38
- "ces_Latn",
39
- "cjk_Latn",
40
- "ckb_Arab",
41
- "crh_Latn",
42
- "cym_Latn",
43
- "dan_Latn",
44
- "deu_Latn",
45
- "dik_Latn",
46
- "dyu_Latn",
47
- "dzo_Tibt",
48
- "ell_Grek",
49
- "eng_Latn",
50
- "epo_Latn",
51
- "est_Latn",
52
- "eus_Latn",
53
- "ewe_Latn",
54
- "fao_Latn",
55
- "pes_Arab",
56
- "fij_Latn",
57
- "fin_Latn",
58
- "fon_Latn",
59
- "fra_Latn",
60
- "fur_Latn",
61
- "fuv_Latn",
62
- "gla_Latn",
63
- "gle_Latn",
64
- "glg_Latn",
65
- "grn_Latn",
66
- "guj_Gujr",
67
- "hat_Latn",
68
- "hau_Latn",
69
- "heb_Hebr",
70
- "hin_Deva",
71
- "hne_Deva",
72
- "hrv_Latn",
73
- "hun_Latn",
74
- "hye_Armn",
75
- "ibo_Latn",
76
- "ilo_Latn",
77
- "ind_Latn",
78
- "isl_Latn",
79
- "ita_Latn",
80
- "jav_Latn",
81
- "jpn_Jpan",
82
- "kab_Latn",
83
- "kac_Latn",
84
- "kam_Latn",
85
- "kan_Knda",
86
- "kas_Arab",
87
- "kas_Deva",
88
- "kat_Geor",
89
- "knc_Arab",
90
- "knc_Latn",
91
- "kaz_Cyrl",
92
- "kbp_Latn",
93
- "kea_Latn",
94
- "khm_Khmr",
95
- "kik_Latn",
96
- "kin_Latn",
97
- "kir_Cyrl",
98
- "kmb_Latn",
99
- "kon_Latn",
100
- "kor_Hang",
101
- "kmr_Latn",
102
- "lao_Laoo",
103
- "lvs_Latn",
104
- "lij_Latn",
105
- "lim_Latn",
106
- "lin_Latn",
107
- "lit_Latn",
108
- "lmo_Latn",
109
- "ltg_Latn",
110
- "ltz_Latn",
111
- "lua_Latn",
112
- "lug_Latn",
113
- "luo_Latn",
114
- "lus_Latn",
115
- "mag_Deva",
116
- "mai_Deva",
117
- "mal_Mlym",
118
- "mar_Deva",
119
- "min_Latn",
120
- "mkd_Cyrl",
121
- "plt_Latn",
122
- "mlt_Latn",
123
- "mni_Beng",
124
- "khk_Cyrl",
125
- "mos_Latn",
126
- "mri_Latn",
127
- "zsm_Latn",
128
- "mya_Mymr",
129
- "nld_Latn",
130
- "nno_Latn",
131
- "nob_Latn",
132
- "npi_Deva",
133
- "nso_Latn",
134
- "nus_Latn",
135
- "nya_Latn",
136
- "oci_Latn",
137
- "gaz_Latn",
138
- "ory_Orya",
139
- "pag_Latn",
140
- "pan_Guru",
141
- "pap_Latn",
142
- "pol_Latn",
143
- "por_Latn",
144
- "prs_Arab",
145
- "pbt_Arab",
146
- "quy_Latn",
147
- "ron_Latn",
148
- "run_Latn",
149
- "rus_Cyrl",
150
- "sag_Latn",
151
- "san_Deva",
152
- "sat_Beng",
153
- "scn_Latn",
154
- "shn_Mymr",
155
- "sin_Sinh",
156
- "slk_Latn",
157
- "slv_Latn",
158
- "smo_Latn",
159
- "sna_Latn",
160
- "snd_Arab",
161
- "som_Latn",
162
- "sot_Latn",
163
- "spa_Latn",
164
- "als_Latn",
165
- "srd_Latn",
166
- "srp_Cyrl",
167
- "ssw_Latn",
168
- "sun_Latn",
169
- "swe_Latn",
170
- "swh_Latn",
171
- "szl_Latn",
172
- "tam_Taml",
173
- "tat_Cyrl",
174
- "tel_Telu",
175
- "tgk_Cyrl",
176
- "tgl_Latn",
177
- "tha_Thai",
178
- "tir_Ethi",
179
- "taq_Latn",
180
- "taq_Tfng",
181
- "tpi_Latn",
182
- "tsn_Latn",
183
- "tso_Latn",
184
- "tuk_Latn",
185
- "tum_Latn",
186
- "tur_Latn",
187
- "twi_Latn",
188
- "tzm_Tfng",
189
- "uig_Arab",
190
- "ukr_Cyrl",
191
- "umb_Latn",
192
- "urd_Arab",
193
- "uzn_Latn",
194
- "vec_Latn",
195
- "vie_Latn",
196
- "war_Latn",
197
- "wol_Latn",
198
- "xho_Latn",
199
- "ydd_Hebr",
200
- "yor_Latn",
201
- "yue_Hant",
202
- "zho_Hans",
203
- "zho_Hant",
204
- "zul_Latn"
205
- ],
206
- "bos_token": "<s>",
207
- "cls_token": "<s>",
208
- "eos_token": "</s>",
209
- "mask_token": {
210
- "content": "<mask>",
211
- "lstrip": true,
212
- "normalized": true,
213
- "rstrip": false,
214
- "single_word": false
215
- },
216
- "pad_token": "<pad>",
217
- "sep_token": "</s>",
218
- "unk_token": "<unk>"
219
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
special_tokens_map.json DELETED
@@ -1,219 +0,0 @@
1
- {
2
- "additional_special_tokens": [
3
- "ace_Arab",
4
- "ace_Latn",
5
- "acm_Arab",
6
- "acq_Arab",
7
- "aeb_Arab",
8
- "afr_Latn",
9
- "ajp_Arab",
10
- "aka_Latn",
11
- "amh_Ethi",
12
- "apc_Arab",
13
- "arb_Arab",
14
- "ars_Arab",
15
- "ary_Arab",
16
- "arz_Arab",
17
- "asm_Beng",
18
- "ast_Latn",
19
- "awa_Deva",
20
- "ayr_Latn",
21
- "azb_Arab",
22
- "azj_Latn",
23
- "bak_Cyrl",
24
- "bam_Latn",
25
- "ban_Latn",
26
- "bel_Cyrl",
27
- "bem_Latn",
28
- "ben_Beng",
29
- "bho_Deva",
30
- "bjn_Arab",
31
- "bjn_Latn",
32
- "bod_Tibt",
33
- "bos_Latn",
34
- "bug_Latn",
35
- "bul_Cyrl",
36
- "cat_Latn",
37
- "ceb_Latn",
38
- "ces_Latn",
39
- "cjk_Latn",
40
- "ckb_Arab",
41
- "crh_Latn",
42
- "cym_Latn",
43
- "dan_Latn",
44
- "deu_Latn",
45
- "dik_Latn",
46
- "dyu_Latn",
47
- "dzo_Tibt",
48
- "ell_Grek",
49
- "eng_Latn",
50
- "epo_Latn",
51
- "est_Latn",
52
- "eus_Latn",
53
- "ewe_Latn",
54
- "fao_Latn",
55
- "pes_Arab",
56
- "fij_Latn",
57
- "fin_Latn",
58
- "fon_Latn",
59
- "fra_Latn",
60
- "fur_Latn",
61
- "fuv_Latn",
62
- "gla_Latn",
63
- "gle_Latn",
64
- "glg_Latn",
65
- "grn_Latn",
66
- "guj_Gujr",
67
- "hat_Latn",
68
- "hau_Latn",
69
- "heb_Hebr",
70
- "hin_Deva",
71
- "hne_Deva",
72
- "hrv_Latn",
73
- "hun_Latn",
74
- "hye_Armn",
75
- "ibo_Latn",
76
- "ilo_Latn",
77
- "ind_Latn",
78
- "isl_Latn",
79
- "ita_Latn",
80
- "jav_Latn",
81
- "jpn_Jpan",
82
- "kab_Latn",
83
- "kac_Latn",
84
- "kam_Latn",
85
- "kan_Knda",
86
- "kas_Arab",
87
- "kas_Deva",
88
- "kat_Geor",
89
- "knc_Arab",
90
- "knc_Latn",
91
- "kaz_Cyrl",
92
- "kbp_Latn",
93
- "kea_Latn",
94
- "khm_Khmr",
95
- "kik_Latn",
96
- "kin_Latn",
97
- "kir_Cyrl",
98
- "kmb_Latn",
99
- "kon_Latn",
100
- "kor_Hang",
101
- "kmr_Latn",
102
- "lao_Laoo",
103
- "lvs_Latn",
104
- "lij_Latn",
105
- "lim_Latn",
106
- "lin_Latn",
107
- "lit_Latn",
108
- "lmo_Latn",
109
- "ltg_Latn",
110
- "ltz_Latn",
111
- "lua_Latn",
112
- "lug_Latn",
113
- "luo_Latn",
114
- "lus_Latn",
115
- "mag_Deva",
116
- "mai_Deva",
117
- "mal_Mlym",
118
- "mar_Deva",
119
- "min_Latn",
120
- "mkd_Cyrl",
121
- "plt_Latn",
122
- "mlt_Latn",
123
- "mni_Beng",
124
- "khk_Cyrl",
125
- "mos_Latn",
126
- "mri_Latn",
127
- "zsm_Latn",
128
- "mya_Mymr",
129
- "nld_Latn",
130
- "nno_Latn",
131
- "nob_Latn",
132
- "npi_Deva",
133
- "nso_Latn",
134
- "nus_Latn",
135
- "nya_Latn",
136
- "oci_Latn",
137
- "gaz_Latn",
138
- "ory_Orya",
139
- "pag_Latn",
140
- "pan_Guru",
141
- "pap_Latn",
142
- "pol_Latn",
143
- "por_Latn",
144
- "prs_Arab",
145
- "pbt_Arab",
146
- "quy_Latn",
147
- "ron_Latn",
148
- "run_Latn",
149
- "rus_Cyrl",
150
- "sag_Latn",
151
- "san_Deva",
152
- "sat_Beng",
153
- "scn_Latn",
154
- "shn_Mymr",
155
- "sin_Sinh",
156
- "slk_Latn",
157
- "slv_Latn",
158
- "smo_Latn",
159
- "sna_Latn",
160
- "snd_Arab",
161
- "som_Latn",
162
- "sot_Latn",
163
- "spa_Latn",
164
- "als_Latn",
165
- "srd_Latn",
166
- "srp_Cyrl",
167
- "ssw_Latn",
168
- "sun_Latn",
169
- "swe_Latn",
170
- "swh_Latn",
171
- "szl_Latn",
172
- "tam_Taml",
173
- "tat_Cyrl",
174
- "tel_Telu",
175
- "tgk_Cyrl",
176
- "tgl_Latn",
177
- "tha_Thai",
178
- "tir_Ethi",
179
- "taq_Latn",
180
- "taq_Tfng",
181
- "tpi_Latn",
182
- "tsn_Latn",
183
- "tso_Latn",
184
- "tuk_Latn",
185
- "tum_Latn",
186
- "tur_Latn",
187
- "twi_Latn",
188
- "tzm_Tfng",
189
- "uig_Arab",
190
- "ukr_Cyrl",
191
- "umb_Latn",
192
- "urd_Arab",
193
- "uzn_Latn",
194
- "vec_Latn",
195
- "vie_Latn",
196
- "war_Latn",
197
- "wol_Latn",
198
- "xho_Latn",
199
- "ydd_Hebr",
200
- "yor_Latn",
201
- "yue_Hant",
202
- "zho_Hans",
203
- "zho_Hant",
204
- "zul_Latn"
205
- ],
206
- "bos_token": "<s>",
207
- "cls_token": "<s>",
208
- "eos_token": "</s>",
209
- "mask_token": {
210
- "content": "<mask>",
211
- "lstrip": true,
212
- "normalized": true,
213
- "rstrip": false,
214
- "single_word": false
215
- },
216
- "pad_token": "<pad>",
217
- "sep_token": "</s>",
218
- "unk_token": "<unk>"
219
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer_config copy.json DELETED
@@ -1,25 +0,0 @@
1
- {
2
- "additional_special_tokens": null,
3
- "bos_token": "<s>",
4
- "clean_up_tokenization_spaces": true,
5
- "cls_token": "<s>",
6
- "eos_token": "</s>",
7
- "legacy_behaviour": false,
8
- "mask_token": {
9
- "__type": "AddedToken",
10
- "content": "<mask>",
11
- "lstrip": true,
12
- "normalized": true,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "model_max_length": 1024,
17
- "pad_token": "<pad>",
18
- "sep_token": "</s>",
19
- "sp_model_kwargs": {},
20
- "src_lang": null,
21
- "tgt_lang": null,
22
- "tokenizer_class": "NllbTokenizer",
23
- "tokenizer_file": null,
24
- "unk_token": "<unk>"
25
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer_config.json DELETED
@@ -1,25 +0,0 @@
1
- {
2
- "additional_special_tokens": null,
3
- "bos_token": "<s>",
4
- "cls_token": "<s>",
5
- "eos_token": "</s>",
6
- "mask_token": {
7
- "__type": "AddedToken",
8
- "content": "<mask>",
9
- "lstrip": true,
10
- "normalized": true,
11
- "rstrip": false,
12
- "single_word": false
13
- },
14
- "model_max_length": 1024,
15
- "name_or_path": "facebook/nllb-200-distilled-1.3B",
16
- "pad_token": "<pad>",
17
- "sep_token": "</s>",
18
- "sp_model_kwargs": {},
19
- "special_tokens_map_file": null,
20
- "src_lang": null,
21
- "tgt_lang": null,
22
- "tokenizer_class": "NllbTokenizer",
23
- "tokenizer_file": null,
24
- "unk_token": "<unk>"
25
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
trainer_state copy.json DELETED
@@ -1,15 +0,0 @@
1
- {
2
- "best_metric": null,
3
- "best_model_checkpoint": null,
4
- "epoch": 0.032,
5
- "global_step": 200,
6
- "is_hyper_param_search": false,
7
- "is_local_process_zero": true,
8
- "is_world_process_zero": true,
9
- "log_history": [],
10
- "max_steps": 1000,
11
- "num_train_epochs": 1,
12
- "total_flos": 584269271728128.0,
13
- "trial_name": null,
14
- "trial_params": null
15
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
trainer_state.json DELETED
@@ -1,15 +0,0 @@
1
- {
2
- "best_metric": null,
3
- "best_model_checkpoint": null,
4
- "epoch": 0.032,
5
- "global_step": 200,
6
- "is_hyper_param_search": false,
7
- "is_local_process_zero": true,
8
- "is_world_process_zero": true,
9
- "log_history": [],
10
- "max_steps": 1000,
11
- "num_train_epochs": 1,
12
- "total_flos": 584269271728128.0,
13
- "trial_name": null,
14
- "trial_params": null
15
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
training_args copy.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:f8e5dc0388daf440f2e481ba6965671a81ada41e5187563c8330b254ed3b29a6
3
- size 4079
 
 
 
 
training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:797c90911657edbb91466351cee6dd61048f82aff22d4f1049b17bc172150252
3
- size 3503