Hugging Face's logo

microsoft
/
Phi-4-multimodal-instruct

Automatic Speech Recognition
Transformers
Safetensors
phi4mm
text-generation
nlp
code
audio
speech-summarization
speech-translation
visual-question-answering
phi-4-multimodal
phi
phi-4-mini
custom_code
Model card Files Community
58

Upload folder using huggingface_hub

#53
by cyrilvallez HF staff - opened
base: refs/heads/main
←
from: refs/pr/53
Discussion Files changed
+32
-2862
Files changed (27) hide show
  1. speech-lora/adapter_config.json → adapter_config.json +16 -10
  2. speech-lora/adapter_model.safetensors → adapter_model.safetensors +2 -2
  3. added_tokens.json +0 -12
  4. config.json +0 -221
  5. generation_config.json +0 -11
  6. model-00001-of-00003.safetensors +0 -3
  7. model-00002-of-00003.safetensors +0 -3
  8. model-00003-of-00003.safetensors +0 -3
  9. model.safetensors.index.json +0 -0
  10. preprocessor_config.json +0 -14
  11. processor_config.json +0 -6
  12. special_tokens_map.json +0 -24
  13. speech-lora/added_tokens.json +0 -12
  14. speech-lora/special_tokens_map.json +0 -24
  15. speech-lora/tokenizer.json +0 -3
  16. speech-lora/tokenizer_config.json +0 -125
  17. speech-lora/vocab.json +0 -0
  18. tokenizer.json +0 -3
  19. tokenizer_config.json +0 -125
  20. vision-lora/adapter_config.json +0 -23
  21. vision-lora/adapter_model.safetensors +0 -3
  22. vision-lora/added_tokens.json +0 -12
  23. vision-lora/special_tokens_map.json +0 -24
  24. vision-lora/tokenizer.json +0 -3
  25. vision-lora/tokenizer_config.json +0 -125
  26. vision-lora/vocab.json +0 -0
  27. vocab.json +0 -0
speech-lora/adapter_config.json → adapter_config.json RENAMED
@@ -1,23 +1,29 @@
1
  {
 
2
  "auto_mapping": null,
3
- "base_model_name_or_path": "TBA",
4
  "bias": "none",
 
 
5
  "fan_in_fan_out": false,
6
- "inference_mode": true,
7
  "init_lora_weights": true,
 
8
  "layers_pattern": null,
9
  "layers_to_transform": null,
 
10
  "lora_alpha": 640,
 
11
  "lora_dropout": 0.01,
12
- "modules_to_save": [],
 
 
13
  "peft_type": "LORA",
14
  "r": 320,
 
15
  "revision": null,
16
- "target_modules": [
17
- "qkv_proj",
18
- "o_proj",
19
- "gate_up_proj",
20
- "down_proj"
21
- ],
22
- "task_type": "CAUSAL_LM"
23
  }
 
1
  {
2
+ "alpha_pattern": {},
3
  "auto_mapping": null,
4
+ "base_model_name_or_path": null,
5
  "bias": "none",
6
+ "eva_config": null,
7
+ "exclude_modules": null,
8
  "fan_in_fan_out": false,
9
+ "inference_mode": false,
10
  "init_lora_weights": true,
11
+ "layer_replication": null,
12
  "layers_pattern": null,
13
  "layers_to_transform": null,
14
+ "loftq_config": {},
15
  "lora_alpha": 640,
16
+ "lora_bias": false,
17
  "lora_dropout": 0.01,
18
+ "megatron_config": null,
19
+ "megatron_core": "megatron.core",
20
+ "modules_to_save": null,
21
  "peft_type": "LORA",
22
  "r": 320,
23
+ "rank_pattern": {},
24
  "revision": null,
25
+ "target_modules": "model.layers.\\d+.((self_attn.(qkv|o)_proj)|(mlp.(gate_up|down)_proj))",
26
+ "task_type": "CAUSAL_LM",
27
+ "use_dora": false,
28
+ "use_rslora": false
 
 
 
29
  }
speech-lora/adapter_model.safetensors → adapter_model.safetensors RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1c2237461a4d1f9292cd128147bd3f0f70326a48d5d79c8e0f7583b26c095b30
3
- size 922782296
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16f70b0aba566f6c30e67a11e90033453e9375d102e031cec40956a2a0e9771e
3
+ size 922777944
added_tokens.json DELETED
@@ -1,12 +0,0 @@
1
- {
2
- "<|/tool_call|>": 200026,
3
- "<|/tool|>": 200024,
4
- "<|assistant|>": 200019,
5
- "<|end|>": 200020,
6
- "<|system|>": 200022,
7
- "<|tag|>": 200028,
8
- "<|tool_call|>": 200025,
9
- "<|tool_response|>": 200027,
10
- "<|tool|>": 200023,
11
- "<|user|>": 200021
12
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
config.json DELETED
@@ -1,221 +0,0 @@
1
- {
2
- "_name_or_path": "Phi-4-multimodal-instruct",
3
- "architectures": [
4
- "Phi4MMForCausalLM"
5
- ],
6
- "attention_bias": false,
7
- "attention_dropout": 0.0,
8
- "audio_processor": {
9
- "config": {
10
- "activation": "swish",
11
- "activation_checkpointing": {
12
- "interval": 1,
13
- "module": "transformer",
14
- "offload": false
15
- },
16
- "attention_dim": 1024,
17
- "attention_heads": 16,
18
- "batch_norm": false,
19
- "bias_in_glu": true,
20
- "causal": true,
21
- "chunk_size": -1,
22
- "cnn_layer_norm": true,
23
- "conv_activation": "swish",
24
- "conv_glu_type": "swish",
25
- "depthwise_multiplier": 1,
26
- "depthwise_seperable_out_channel": 1024,
27
- "dropout_rate": 0.0,
28
- "encoder_embedding_config": {
29
- "input_size": 80
30
- },
31
- "ext_pw_kernel_size": 1,
32
- "ext_pw_out_channel": 1024,
33
- "input_layer": "nemo_conv",
34
- "input_size": 80,
35
- "kernel_size": 3,
36
- "left_chunk": 18,
37
- "linear_units": 1536,
38
- "nemo_conv_settings": {
39
- "conv_channels": 1024
40
- },
41
- "num_blocks": 24,
42
- "relative_attention_bias_args": {
43
- "t5_bias_max_distance": 500,
44
- "type": "t5"
45
- },
46
- "time_reduction": 8
47
- },
48
- "name": "cascades"
49
- },
50
- "auto_map": {
51
- "AutoConfig": "configuration_phi4mm.Phi4MMConfig",
52
- "AutoModelForCausalLM": "modeling_phi4mm.Phi4MMForCausalLM",
53
- "AutoTokenizer": "Xenova/gpt-4o"
54
- },
55
- "bos_token_id": 199999,
56
- "embd_layer": {
57
- "audio_embd_layer": {
58
- "compression_rate": 8,
59
- "downsample_rate": 1,
60
- "embedding_cls": "audio",
61
- "enable_gradient_checkpointing": true,
62
- "projection_cls": "mlp",
63
- "use_conv_downsample": false,
64
- "use_qformer": false
65
- },
66
- "embedding_cls": "image_audio",
67
- "image_embd_layer": {
68
- "crop_size": 448,
69
- "embedding_cls": "tune_image",
70
- "enable_gradient_checkpointing": true,
71
- "hd_transform_order": "sub_glb",
72
- "image_token_compression_cls": "avg_pool_2d",
73
- "projection_cls": "mlp",
74
- "use_hd_transform": true,
75
- "with_learnable_separator": true
76
- }
77
- },
78
- "embd_pdrop": 0.0,
79
- "eos_token_id": 199999,
80
- "full_attn_mod": 1,
81
- "hidden_act": "silu",
82
- "hidden_size": 3072,
83
- "initializer_range": 0.02,
84
- "intermediate_size": 8192,
85
- "interpolate_factor": 1,
86
- "lm_head_bias": false,
87
- "vision_lora": {
88
- "dp": 0.0,
89
- "layer": "layers.*((self_attn\\.(qkv_proj|o_proj))|(mlp\\.(gate_up|down)_proj))",
90
- "lora_alpha": 512,
91
- "r": 256
92
- },
93
- "speech_lora": {
94
- "dp": 0.01,
95
- "layer": "((layers.*self_attn\\.(qkv|o)_proj)|(layers.*mlp\\.(gate_up|down)_proj))",
96
- "lora_alpha": 640,
97
- "r": 320
98
- },
99
- "max_position_embeddings": 131072,
100
- "mlp_bias": false,
101
- "model_type": "phi4mm",
102
- "num_attention_heads": 24,
103
- "num_hidden_layers": 32,
104
- "num_key_value_heads": 8,
105
- "original_max_position_embeddings": 4096,
106
- "pad_token_id": 199999,
107
- "partial_rotary_factor": 0.75,
108
- "resid_pdrop": 0.0,
109
- "rms_norm_eps": 1e-05,
110
- "rope_scaling": {
111
- "long_factor": [
112
- 1,
113
- 1.118320672,
114
- 1.250641126,
115
- 1.398617824,
116
- 1.564103225,
117
- 1.74916897,
118
- 1.956131817,
119
- 2.187582649,
120
- 2.446418898,
121
- 2.735880826,
122
- 3.059592084,
123
- 3.421605075,
124
- 3.826451687,
125
- 4.279200023,
126
- 4.785517845,
127
- 5.351743533,
128
- 5.984965424,
129
- 6.693110555,
130
- 7.485043894,
131
- 8.370679318,
132
- 9.36110372,
133
- 10.4687158,
134
- 11.70738129,
135
- 13.09260651,
136
- 14.64173252,
137
- 16.37415215,
138
- 18.31155283,
139
- 20.47818807,
140
- 22.90118105,
141
- 25.61086418,
142
- 28.64115884,
143
- 32.03,
144
- 32.1,
145
- 32.13,
146
- 32.23,
147
- 32.6,
148
- 32.61,
149
- 32.64,
150
- 32.66,
151
- 32.7,
152
- 32.71,
153
- 32.93,
154
- 32.97,
155
- 33.28,
156
- 33.49,
157
- 33.5,
158
- 44.16,
159
- 47.77
160
- ],
161
- "short_factor": [
162
- 1.0,
163
- 1.0,
164
- 1.0,
165
- 1.0,
166
- 1.0,
167
- 1.0,
168
- 1.0,
169
- 1.0,
170
- 1.0,
171
- 1.0,
172
- 1.0,
173
- 1.0,
174
- 1.0,
175
- 1.0,
176
- 1.0,
177
- 1.0,
178
- 1.0,
179
- 1.0,
180
- 1.0,
181
- 1.0,
182
- 1.0,
183
- 1.0,
184
- 1.0,
185
- 1.0,
186
- 1.0,
187
- 1.0,
188
- 1.0,
189
- 1.0,
190
- 1.0,
191
- 1.0,
192
- 1.0,
193
- 1.0,
194
- 1.0,
195
- 1.0,
196
- 1.0,
197
- 1.0,
198
- 1.0,
199
- 1.0,
200
- 1.0,
201
- 1.0,
202
- 1.0,
203
- 1.0,
204
- 1.0,
205
- 1.0,
206
- 1.0,
207
- 1.0,
208
- 1.0,
209
- 1.0
210
- ],
211
- "type": "longrope"
212
- },
213
- "rope_theta": 10000.0,
214
- "sliding_window": 262144,
215
- "tie_word_embeddings": true,
216
- "torch_dtype": "bfloat16",
217
- "transformers_version": "4.46.1",
218
- "use_cache": true,
219
- "vocab_size": 200064,
220
- "_attn_implementation": "flash_attention_2"
221
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
generation_config.json DELETED
@@ -1,11 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 199999,
4
- "eos_token_id": [
5
- 200020,
6
- 199999
7
- ],
8
- "pad_token_id": 199999,
9
- "transformers_version": "4.46.1",
10
- "use_cache": true
11
- }
 
 
 
 
 
 
 
 
 
 
 
 
model-00001-of-00003.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:c46bb03332d82f6a3eaf85bd20af388dd4d4d68b198c2203c965c7381a466094
3
- size 4997504848
 
 
 
 
model-00002-of-00003.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b3e812c0c8acef4e7f5e34d6c9f77a7640ee4a2b93ea351921365ac62f19918d
3
- size 4952333128
 
 
 
 
model-00003-of-00003.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:7be96b7339303752634b202d3f377bcf312a03046586eca6cea23347ace1e65a
3
- size 1199389232
 
 
 
 
model.safetensors.index.json DELETED
The diff for this file is too large to render. See raw diff
 
preprocessor_config.json DELETED
@@ -1,14 +0,0 @@
1
- {
2
- "auto_map": {
3
- "AutoProcessor": "processing_phi4mm.Phi4MMProcessor",
4
- "AutoImageProcessor": "processing_phi4mm.Phi4MMImageProcessor",
5
- "AutoFeatureExtractor": "processing_phi4mm.Phi4MMAudioFeatureExtractor"
6
- },
7
- "image_processor_type": "Phi4MMImageProcessor",
8
- "processor_class": "Phi4MMProcessor",
9
- "feature_extractor_type": "Phi4MMAudioFeatureExtractor",
10
- "audio_compression_rate": 8,
11
- "audio_downsample_rate": 1,
12
- "audio_feat_stride": 1,
13
- "dynamic_hd": 36
14
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
processor_config.json DELETED
@@ -1,6 +0,0 @@
1
- {
2
- "auto_map": {
3
- "AutoProcessor": "processing_phi4mm.Phi4MMProcessor"
4
- },
5
- "processor_class": "Phi4MMProcessor"
6
- }
 
 
 
 
 
 
 
special_tokens_map.json DELETED
@@ -1,24 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<|endoftext|>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|endoftext|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": "<|endoftext|>",
17
- "unk_token": {
18
- "content": "<|endoftext|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- }
24
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
speech-lora/added_tokens.json DELETED
@@ -1,12 +0,0 @@
1
- {
2
- "<|/tool_call|>": 200026,
3
- "<|/tool|>": 200024,
4
- "<|assistant|>": 200019,
5
- "<|end|>": 200020,
6
- "<|system|>": 200022,
7
- "<|tag|>": 200028,
8
- "<|tool_call|>": 200025,
9
- "<|tool_response|>": 200027,
10
- "<|tool|>": 200023,
11
- "<|user|>": 200021
12
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
speech-lora/special_tokens_map.json DELETED
@@ -1,24 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<|endoftext|>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|endoftext|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": "<|endoftext|>",
17
- "unk_token": {
18
- "content": "<|endoftext|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- }
24
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
speech-lora/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea
3
- size 15524095
 
 
 
 
speech-lora/tokenizer_config.json DELETED
@@ -1,125 +0,0 @@
1
- {
2
- "add_prefix_space": false,
3
- "added_tokens_decoder": {
4
- "200010": {
5
- "content": "<|endoftext10|>",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": true
11
- },
12
- "200011": {
13
- "content": "<|endoftext11|>",
14
- "lstrip": false,
15
- "normalized": false,
16
- "rstrip": false,
17
- "single_word": false,
18
- "special": true
19
- },
20
- "199999": {
21
- "content": "<|endoftext|>",
22
- "lstrip": false,
23
- "normalized": false,
24
- "rstrip": false,
25
- "single_word": false,
26
- "special": true
27
- },
28
- "200018": {
29
- "content": "<|endofprompt|>",
30
- "lstrip": false,
31
- "normalized": false,
32
- "rstrip": false,
33
- "single_word": false,
34
- "special": true
35
- },
36
- "200019": {
37
- "content": "<|assistant|>",
38
- "lstrip": false,
39
- "normalized": false,
40
- "rstrip": true,
41
- "single_word": false,
42
- "special": true
43
- },
44
- "200020": {
45
- "content": "<|end|>",
46
- "lstrip": false,
47
- "normalized": false,
48
- "rstrip": true,
49
- "single_word": false,
50
- "special": true
51
- },
52
- "200021": {
53
- "content": "<|user|>",
54
- "lstrip": false,
55
- "normalized": false,
56
- "rstrip": true,
57
- "single_word": false,
58
- "special": true
59
- },
60
- "200022": {
61
- "content": "<|system|>",
62
- "lstrip": false,
63
- "normalized": false,
64
- "rstrip": true,
65
- "single_word": false,
66
- "special": true
67
- },
68
- "200023": {
69
- "content": "<|tool|>",
70
- "lstrip": false,
71
- "normalized": false,
72
- "rstrip": true,
73
- "single_word": false,
74
- "special": false
75
- },
76
- "200024": {
77
- "content": "<|/tool|>",
78
- "lstrip": false,
79
- "normalized": false,
80
- "rstrip": true,
81
- "single_word": false,
82
- "special": false
83
- },
84
- "200025": {
85
- "content": "<|tool_call|>",
86
- "lstrip": false,
87
- "normalized": false,
88
- "rstrip": true,
89
- "single_word": false,
90
- "special": false
91
- },
92
- "200026": {
93
- "content": "<|/tool_call|>",
94
- "lstrip": false,
95
- "normalized": false,
96
- "rstrip": true,
97
- "single_word": false,
98
- "special": false
99
- },
100
- "200027": {
101
- "content": "<|tool_response|>",
102
- "lstrip": false,
103
- "normalized": false,
104
- "rstrip": true,
105
- "single_word": false,
106
- "special": false
107
- },
108
- "200028": {
109
- "content": "<|tag|>",
110
- "lstrip": false,
111
- "normalized": false,
112
- "rstrip": true,
113
- "single_word": false,
114
- "special": true
115
- }
116
- },
117
- "bos_token": "<|endoftext|>",
118
- "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}",
119
- "clean_up_tokenization_spaces": false,
120
- "eos_token": "<|endoftext|>",
121
- "model_max_length": 128000,
122
- "pad_token": "<|endoftext|>",
123
- "tokenizer_class": "GPT2TokenizerFast",
124
- "unk_token": "<|endoftext|>"
125
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
speech-lora/vocab.json DELETED
The diff for this file is too large to render. See raw diff
 
tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4c1b9f641d4f8b7247b8d5007dd3b6a9f6a87cb5123134fe0d326f14d10c0585
3
- size 15524479
 
 
 
 
tokenizer_config.json DELETED
@@ -1,125 +0,0 @@
1
- {
2
- "add_prefix_space": false,
3
- "added_tokens_decoder": {
4
- "200010": {
5
- "content": "<|endoftext10|>",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": true
11
- },
12
- "200011": {
13
- "content": "<|endoftext11|>",
14
- "lstrip": false,
15
- "normalized": false,
16
- "rstrip": false,
17
- "single_word": false,
18
- "special": true
19
- },
20
- "199999": {
21
- "content": "<|endoftext|>",
22
- "lstrip": false,
23
- "normalized": false,
24
- "rstrip": false,
25
- "single_word": false,
26
- "special": true
27
- },
28
- "200018": {
29
- "content": "<|endofprompt|>",
30
- "lstrip": false,
31
- "normalized": false,
32
- "rstrip": false,
33
- "single_word": false,
34
- "special": true
35
- },
36
- "200019": {
37
- "content": "<|assistant|>",
38
- "lstrip": false,
39
- "normalized": false,
40
- "rstrip": true,
41
- "single_word": false,
42
- "special": true
43
- },
44
- "200020": {
45
- "content": "<|end|>",
46
- "lstrip": false,
47
- "normalized": false,
48
- "rstrip": true,
49
- "single_word": false,
50
- "special": true
51
- },
52
- "200021": {
53
- "content": "<|user|>",
54
- "lstrip": false,
55
- "normalized": false,
56
- "rstrip": true,
57
- "single_word": false,
58
- "special": true
59
- },
60
- "200022": {
61
- "content": "<|system|>",
62
- "lstrip": false,
63
- "normalized": false,
64
- "rstrip": true,
65
- "single_word": false,
66
- "special": true
67
- },
68
- "200023": {
69
- "content": "<|tool|>",
70
- "lstrip": false,
71
- "normalized": false,
72
- "rstrip": true,
73
- "single_word": false,
74
- "special": false
75
- },
76
- "200024": {
77
- "content": "<|/tool|>",
78
- "lstrip": false,
79
- "normalized": false,
80
- "rstrip": true,
81
- "single_word": false,
82
- "special": false
83
- },
84
- "200025": {
85
- "content": "<|tool_call|>",
86
- "lstrip": false,
87
- "normalized": false,
88
- "rstrip": true,
89
- "single_word": false,
90
- "special": false
91
- },
92
- "200026": {
93
- "content": "<|/tool_call|>",
94
- "lstrip": false,
95
- "normalized": false,
96
- "rstrip": true,
97
- "single_word": false,
98
- "special": false
99
- },
100
- "200027": {
101
- "content": "<|tool_response|>",
102
- "lstrip": false,
103
- "normalized": false,
104
- "rstrip": true,
105
- "single_word": false,
106
- "special": false
107
- },
108
- "200028": {
109
- "content": "<|tag|>",
110
- "lstrip": false,
111
- "normalized": false,
112
- "rstrip": true,
113
- "single_word": false,
114
- "special": true
115
- }
116
- },
117
- "bos_token": "<|endoftext|>",
118
- "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}",
119
- "clean_up_tokenization_spaces": false,
120
- "eos_token": "<|endoftext|>",
121
- "model_max_length": 131072,
122
- "pad_token": "<|endoftext|>",
123
- "tokenizer_class": "GPT2TokenizerFast",
124
- "unk_token": "<|endoftext|>"
125
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
vision-lora/adapter_config.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "auto_mapping": null,
3
- "base_model_name_or_path": "TBA",
4
- "bias": "none",
5
- "fan_in_fan_out": false,
6
- "inference_mode": true,
7
- "init_lora_weights": true,
8
- "layers_pattern": null,
9
- "layers_to_transform": null,
10
- "lora_alpha": 512,
11
- "lora_dropout": 0.0,
12
- "modules_to_save": [],
13
- "peft_type": "LORA",
14
- "r": 256,
15
- "revision": null,
16
- "target_modules": [
17
- "qkv_proj",
18
- "o_proj",
19
- "gate_up_proj",
20
- "down_proj"
21
- ],
22
- "task_type": "CAUSAL_LM"
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
vision-lora/adapter_model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1620b16722edf701038bf66e3cd46412c7cc5458e58df89e9f92cedb71fcbde8
3
- size 738232904
 
 
 
 
vision-lora/added_tokens.json DELETED
@@ -1,12 +0,0 @@
1
- {
2
- "<|/tool_call|>": 200026,
3
- "<|/tool|>": 200024,
4
- "<|assistant|>": 200019,
5
- "<|end|>": 200020,
6
- "<|system|>": 200022,
7
- "<|tag|>": 200028,
8
- "<|tool_call|>": 200025,
9
- "<|tool_response|>": 200027,
10
- "<|tool|>": 200023,
11
- "<|user|>": 200021
12
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
vision-lora/special_tokens_map.json DELETED
@@ -1,24 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<|endoftext|>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|endoftext|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "pad_token": "<|endoftext|>",
17
- "unk_token": {
18
- "content": "<|endoftext|>",
19
- "lstrip": false,
20
- "normalized": false,
21
- "rstrip": false,
22
- "single_word": false
23
- }
24
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
vision-lora/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea
3
- size 15524095
 
 
 
 
vision-lora/tokenizer_config.json DELETED
@@ -1,125 +0,0 @@
1
- {
2
- "add_prefix_space": false,
3
- "added_tokens_decoder": {
4
- "200010": {
5
- "content": "<|endoftext10|>",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": true
11
- },
12
- "200011": {
13
- "content": "<|endoftext11|>",
14
- "lstrip": false,
15
- "normalized": false,
16
- "rstrip": false,
17
- "single_word": false,
18
- "special": true
19
- },
20
- "199999": {
21
- "content": "<|endoftext|>",
22
- "lstrip": false,
23
- "normalized": false,
24
- "rstrip": false,
25
- "single_word": false,
26
- "special": true
27
- },
28
- "200018": {
29
- "content": "<|endofprompt|>",
30
- "lstrip": false,
31
- "normalized": false,
32
- "rstrip": false,
33
- "single_word": false,
34
- "special": true
35
- },
36
- "200019": {
37
- "content": "<|assistant|>",
38
- "lstrip": false,
39
- "normalized": false,
40
- "rstrip": true,
41
- "single_word": false,
42
- "special": true
43
- },
44
- "200020": {
45
- "content": "<|end|>",
46
- "lstrip": false,
47
- "normalized": false,
48
- "rstrip": true,
49
- "single_word": false,
50
- "special": true
51
- },
52
- "200021": {
53
- "content": "<|user|>",
54
- "lstrip": false,
55
- "normalized": false,
56
- "rstrip": true,
57
- "single_word": false,
58
- "special": true
59
- },
60
- "200022": {
61
- "content": "<|system|>",
62
- "lstrip": false,
63
- "normalized": false,
64
- "rstrip": true,
65
- "single_word": false,
66
- "special": true
67
- },
68
- "200023": {
69
- "content": "<|tool|>",
70
- "lstrip": false,
71
- "normalized": false,
72
- "rstrip": true,
73
- "single_word": false,
74
- "special": false
75
- },
76
- "200024": {
77
- "content": "<|/tool|>",
78
- "lstrip": false,
79
- "normalized": false,
80
- "rstrip": true,
81
- "single_word": false,
82
- "special": false
83
- },
84
- "200025": {
85
- "content": "<|tool_call|>",
86
- "lstrip": false,
87
- "normalized": false,
88
- "rstrip": true,
89
- "single_word": false,
90
- "special": false
91
- },
92
- "200026": {
93
- "content": "<|/tool_call|>",
94
- "lstrip": false,
95
- "normalized": false,
96
- "rstrip": true,
97
- "single_word": false,
98
- "special": false
99
- },
100
- "200027": {
101
- "content": "<|tool_response|>",
102
- "lstrip": false,
103
- "normalized": false,
104
- "rstrip": true,
105
- "single_word": false,
106
- "special": false
107
- },
108
- "200028": {
109
- "content": "<|tag|>",
110
- "lstrip": false,
111
- "normalized": false,
112
- "rstrip": true,
113
- "single_word": false,
114
- "special": true
115
- }
116
- },
117
- "bos_token": "<|endoftext|>",
118
- "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}",
119
- "clean_up_tokenization_spaces": false,
120
- "eos_token": "<|endoftext|>",
121
- "model_max_length": 128000,
122
- "pad_token": "<|endoftext|>",
123
- "tokenizer_class": "GPT2TokenizerFast",
124
- "unk_token": "<|endoftext|>"
125
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
vision-lora/vocab.json DELETED
The diff for this file is too large to render. See raw diff
 
vocab.json DELETED
The diff for this file is too large to render. See raw diff