Hugging Face's logo

Uni-MoE
/
Uni-MoE-v2-e2

Text Generation
Transformers
llava_llama
Inference Endpoints
Model card Files Community
Uni-MoE commited on
Commit
7ed4bcc
·
verified ·
1 Parent(s): dfe0c83

Upload 18 files

Browse files
Files changed (18) hide show
  1. 0_mlp_experts.bin +3 -0
  2. 1_mlp_experts.bin +3 -0
  3. 2_mlp_experts.bin +3 -0
  4. 3_mlp_experts.bin +3 -0
  5. 4_mlp_experts.bin +3 -0
  6. 5_mlp_experts.bin +3 -0
  7. 6_mlp_experts.bin +3 -0
  8. 7_mlp_experts.bin +3 -0
  9. adapter_config.json +147 -0
  10. adapter_model.bin +3 -0
  11. config.json +64 -0
  12. mlp_gates.bin +3 -0
  13. mm_audio_aligner.bin +3 -0
  14. non_lora_trainables.bin +3 -0
  15. special_tokens_map.json +24 -0
  16. tokenizer.model +3 -0
  17. tokenizer_config.json +35 -0
  18. training_args.bin +3 -0
0_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab01094f66514d0101b0fdbe6ba9f7872d7091d189a10fa37b47516c81802954
3
+ size 23268481
1_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1d920ba2a8990855f2927f51b69febae22c9bbb140599f9c49630e09d4174c6
3
+ size 23268481
2_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87d91cc0eb1d54c16d7dfbebfa792886d02526d01c717db134cf0a2d1eb2ae5f
3
+ size 23268481
3_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65d788e37b89e9d48fa8eab2a6a5044b44a551387b523bdfd46d4c7ebb8a1b5a
3
+ size 23268481
4_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fbfaf8aeed94bcf4d46541d49bffa60670cc4119771943ed463779cec1c117ed
3
+ size 23268481
5_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2125d9a6a593d5397498a336ac12dc4ec75938d32b87e093227a7e95c43a0006
3
+ size 23268481
6_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d730ca35fdc7b1dfdeb5e3e428c2b3f68d58a384d07486db35aa8248fd8fd1f6
3
+ size 23268481
7_mlp_experts.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97a1d1f4ad585a4174d4073a1f28a304aae93078b1ca3deb1ac85fa540af24ff
3
+ size 23268481
adapter_config.json ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "checkpoints/Uni-MoE-speech-base",
4
+ "bias": "none",
5
+ "fan_in_fan_out": false,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 16,
11
+ "lora_dropout": 0.05,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "model.layers.17.self_attn.v_proj",
18
+ "model.layers.7.self_attn.o_proj",
19
+ "model.layers.26.self_attn.k_proj",
20
+ "model.layers.23.self_attn.o_proj",
21
+ "model.layers.28.self_attn.o_proj",
22
+ "model.layers.6.self_attn.q_proj",
23
+ "model.layers.22.self_attn.o_proj",
24
+ "model.layers.15.self_attn.o_proj",
25
+ "model.layers.29.self_attn.q_proj",
26
+ "model.layers.5.self_attn.v_proj",
27
+ "model.layers.16.self_attn.k_proj",
28
+ "model.layers.18.self_attn.k_proj",
29
+ "model.layers.3.self_attn.v_proj",
30
+ "model.layers.19.self_attn.v_proj",
31
+ "model.layers.7.self_attn.v_proj",
32
+ "model.layers.10.self_attn.v_proj",
33
+ "model.layers.3.self_attn.q_proj",
34
+ "model.layers.21.self_attn.q_proj",
35
+ "model.layers.31.self_attn.q_proj",
36
+ "model.layers.1.self_attn.v_proj",
37
+ "model.layers.13.self_attn.o_proj",
38
+ "model.layers.2.self_attn.k_proj",
39
+ "model.layers.12.self_attn.q_proj",
40
+ "model.layers.18.self_attn.o_proj",
41
+ "model.layers.24.self_attn.v_proj",
42
+ "model.layers.9.self_attn.o_proj",
43
+ "model.layers.11.self_attn.v_proj",
44
+ "model.layers.15.self_attn.q_proj",
45
+ "model.layers.8.self_attn.v_proj",
46
+ "model.layers.25.self_attn.o_proj",
47
+ "model.layers.24.self_attn.o_proj",
48
+ "model.layers.20.self_attn.v_proj",
49
+ "model.layers.27.self_attn.v_proj",
50
+ "model.layers.16.self_attn.v_proj",
51
+ "model.layers.10.self_attn.k_proj",
52
+ "model.layers.11.self_attn.q_proj",
53
+ "model.layers.17.self_attn.k_proj",
54
+ "model.layers.11.self_attn.o_proj",
55
+ "model.layers.4.self_attn.o_proj",
56
+ "model.layers.28.self_attn.k_proj",
57
+ "model.layers.23.self_attn.v_proj",
58
+ "model.layers.5.self_attn.o_proj",
59
+ "model.layers.8.self_attn.o_proj",
60
+ "model.layers.14.self_attn.v_proj",
61
+ "model.layers.14.self_attn.k_proj",
62
+ "model.layers.7.self_attn.q_proj",
63
+ "model.layers.1.self_attn.k_proj",
64
+ "model.layers.30.self_attn.v_proj",
65
+ "model.layers.22.self_attn.q_proj",
66
+ "model.layers.12.self_attn.k_proj",
67
+ "model.layers.25.self_attn.k_proj",
68
+ "model.layers.16.self_attn.q_proj",
69
+ "model.layers.25.self_attn.v_proj",
70
+ "model.layers.25.self_attn.q_proj",
71
+ "model.layers.19.self_attn.o_proj",
72
+ "model.layers.29.self_attn.k_proj",
73
+ "model.layers.3.self_attn.k_proj",
74
+ "model.layers.5.self_attn.k_proj",
75
+ "model.layers.6.self_attn.o_proj",
76
+ "model.layers.29.self_attn.o_proj",
77
+ "model.layers.9.self_attn.q_proj",
78
+ "model.layers.23.self_attn.k_proj",
79
+ "model.layers.4.self_attn.k_proj",
80
+ "model.layers.11.self_attn.k_proj",
81
+ "model.layers.17.self_attn.o_proj",
82
+ "model.layers.27.self_attn.k_proj",
83
+ "model.layers.1.self_attn.q_proj",
84
+ "model.layers.21.self_attn.v_proj",
85
+ "model.layers.12.self_attn.o_proj",
86
+ "model.layers.7.self_attn.k_proj",
87
+ "model.layers.6.self_attn.k_proj",
88
+ "model.layers.0.self_attn.q_proj",
89
+ "model.layers.0.self_attn.k_proj",
90
+ "model.layers.22.self_attn.k_proj",
91
+ "model.layers.15.self_attn.v_proj",
92
+ "model.layers.13.self_attn.k_proj",
93
+ "model.layers.3.self_attn.o_proj",
94
+ "model.layers.4.self_attn.q_proj",
95
+ "model.layers.5.self_attn.q_proj",
96
+ "model.layers.26.self_attn.q_proj",
97
+ "model.layers.16.self_attn.o_proj",
98
+ "model.layers.21.self_attn.o_proj",
99
+ "model.layers.13.self_attn.q_proj",
100
+ "model.layers.30.self_attn.k_proj",
101
+ "model.layers.31.self_attn.v_proj",
102
+ "model.layers.22.self_attn.v_proj",
103
+ "model.layers.31.self_attn.o_proj",
104
+ "model.layers.24.self_attn.q_proj",
105
+ "model.layers.28.self_attn.q_proj",
106
+ "model.layers.9.self_attn.v_proj",
107
+ "model.layers.1.self_attn.o_proj",
108
+ "model.layers.14.self_attn.q_proj",
109
+ "model.layers.29.self_attn.v_proj",
110
+ "model.layers.27.self_attn.q_proj",
111
+ "model.layers.13.self_attn.v_proj",
112
+ "model.layers.24.self_attn.k_proj",
113
+ "model.layers.2.self_attn.o_proj",
114
+ "model.layers.20.self_attn.o_proj",
115
+ "model.layers.27.self_attn.o_proj",
116
+ "model.layers.20.self_attn.k_proj",
117
+ "model.layers.2.self_attn.q_proj",
118
+ "model.layers.18.self_attn.q_proj",
119
+ "model.layers.6.self_attn.v_proj",
120
+ "model.layers.28.self_attn.v_proj",
121
+ "model.layers.17.self_attn.q_proj",
122
+ "model.layers.19.self_attn.q_proj",
123
+ "model.layers.0.self_attn.v_proj",
124
+ "model.layers.9.self_attn.k_proj",
125
+ "model.layers.8.self_attn.k_proj",
126
+ "model.layers.15.self_attn.k_proj",
127
+ "model.layers.26.self_attn.v_proj",
128
+ "model.layers.2.self_attn.v_proj",
129
+ "model.layers.21.self_attn.k_proj",
130
+ "model.layers.10.self_attn.q_proj",
131
+ "model.layers.30.self_attn.o_proj",
132
+ "model.layers.12.self_attn.v_proj",
133
+ "model.layers.14.self_attn.o_proj",
134
+ "model.layers.19.self_attn.k_proj",
135
+ "model.layers.10.self_attn.o_proj",
136
+ "model.layers.30.self_attn.q_proj",
137
+ "model.layers.31.self_attn.k_proj",
138
+ "model.layers.8.self_attn.q_proj",
139
+ "model.layers.18.self_attn.v_proj",
140
+ "model.layers.26.self_attn.o_proj",
141
+ "model.layers.20.self_attn.q_proj",
142
+ "model.layers.23.self_attn.q_proj",
143
+ "model.layers.4.self_attn.v_proj",
144
+ "model.layers.0.self_attn.o_proj"
145
+ ],
146
+ "task_type": "CAUSAL_LM"
147
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:835479aefeb34eaca9df317a569e34a3b0caaeaefbd6835584e0f87ae6285693
3
+ size 16865677
config.json ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "checkpoints/Uni-MoE-speech-base",
3
+ "architectures": [
4
+ "LlavaLlamaForCausalLM"
5
+ ],
6
+ "audio_projector_type": "linear",
7
+ "bos_token_id": 1,
8
+ "capacity_factor": 1.5,
9
+ "eos_token_id": 2,
10
+ "ep_size": 8,
11
+ "freeze_mm_mlp_adapter": false,
12
+ "freeze_mm_vision_resampler": false,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 4096,
15
+ "image_aspect_ratio": "square",
16
+ "image_grid_pinpoints": null,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 11008,
19
+ "language": "English",
20
+ "local_files_only": false,
21
+ "lora_alpha": 16,
22
+ "lora_dropout": 0.05,
23
+ "lora_r": 8,
24
+ "max_length": 4096,
25
+ "max_position_embeddings": 4096,
26
+ "mm_audio_hidden_size": 768,
27
+ "mm_audio_select_feature": "patch",
28
+ "mm_audio_select_layer": 0,
29
+ "mm_audio_tower": "checkpoints/whisper-small",
30
+ "mm_hidden_size": 1024,
31
+ "mm_projector_lr": null,
32
+ "mm_projector_type": "mlp2x_gelu",
33
+ "mm_query_tokens": true,
34
+ "mm_resampler_type": null,
35
+ "mm_use_im_patch_token": false,
36
+ "mm_use_im_start_end": false,
37
+ "mm_vision_select_feature": "patch",
38
+ "mm_vision_select_layer": -2,
39
+ "mm_vision_tower": "checkpoints/clip-vit-large-patch14-336",
40
+ "model_type": "llava_llama",
41
+ "moe": true,
42
+ "moe_type": "dense",
43
+ "num_attention_heads": 32,
44
+ "num_experts": 8,
45
+ "num_experts_per_tok": 2,
46
+ "num_hidden_layers": 32,
47
+ "num_key_value_heads": 32,
48
+ "pad_token_id": 0,
49
+ "pretraining_tp": 1,
50
+ "query_tokens_size": 50,
51
+ "rms_norm_eps": 1e-05,
52
+ "rope_scaling": null,
53
+ "task": "transcribe",
54
+ "tie_word_embeddings": false,
55
+ "torch_dtype": "float32",
56
+ "transformers_version": "4.31.0",
57
+ "tune_mm_audio_aligner": true,
58
+ "tune_mm_mlp_adapter": true,
59
+ "tune_mm_vision_resampler": false,
60
+ "unfreeze_mm_vision_tower": false,
61
+ "use_cache": true,
62
+ "use_mm_proj": true,
63
+ "vocab_size": 32000
64
+ }
mlp_gates.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6fb75bb523ebc130e57fc350f0fc14cc6698c2a19f69753b40d703204aa521b2
3
+ size 2107337
mm_audio_aligner.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56ee55e712bfc17fb48b1c1628b087f349fcd011d257ae6c0668b91ad4830063
3
+ size 313644388
non_lora_trainables.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17b0687ed60f3b9e1e9ff0b3ccc576ad5d2fb47583ca1a1099bed46edcf582b0
3
+ size 41961191
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<unk>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<s>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "clean_up_tokenization_spaces": false,
13
+ "eos_token": {
14
+ "__type": "AddedToken",
15
+ "content": "</s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "legacy": false,
22
+ "model_max_length": 2048,
23
+ "pad_token": null,
24
+ "padding_side": "right",
25
+ "sp_model_kwargs": {},
26
+ "tokenizer_class": "LlamaTokenizer",
27
+ "unk_token": {
28
+ "__type": "AddedToken",
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ }
35
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fd4e76d66bd8ef23cc5be0b00ea7105f0742a5a6c43d6aaa9654b4882f6f5ded
3
+ size 5691