update v1
Browse files- README.md +6 -5
- config.json +1 -1
- configuration_imp.py +1 -1
- requirements.txt +0 -4
README.md
CHANGED
@@ -5,9 +5,9 @@ pipeline_tag: visual-question-answering
|
|
5 |
# 😈Imp
|
6 |
|
7 |
|
8 |
-
The Imp project aims to provide a family of a strong multimodal `small` language models (MSLMs). Our `imp-
|
9 |
|
10 |
-
As shown in the Table below, `imp-
|
11 |
|
12 |
We release our model weights and provide an example below to run our model . Detailed technical report and corresponding training/evaluation code will be released soon on our [GitHub repo](https://github.com/MILVLG/imp). We will persistently improve our model and release the next versions to further improve model performance :)
|
13 |
|
@@ -25,11 +25,11 @@ torch.set_default_device("cuda")
|
|
25 |
|
26 |
#Create model
|
27 |
model = AutoModelForCausalLM.from_pretrained(
|
28 |
-
"MILVLG/imp-
|
29 |
torch_dtype=torch.float16,
|
30 |
device_map="auto",
|
31 |
trust_remote_code=True)
|
32 |
-
tokenizer = AutoTokenizer.from_pretrained("MILVLG/imp-
|
33 |
|
34 |
#Set inputs
|
35 |
text = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: <image>\nWhat are the colors of the bus in the image? ASSISTANT:"
|
@@ -57,9 +57,10 @@ We conduct evaluation on 9 commonly-used benchmarks, including 5 academic VQA be
|
|
57 |
| [LLaVA-Phi](https://github.com/zhuyiche/llava-phi) | 3B | 71.40 | - | 35.90 | 68.40 | 48.60 | 85.00 | 1335.1 | 59.80 |28.9|
|
58 |
| [MobileVLM](https://huggingface.co/mtgv/MobileVLM-3B) | 3B | - | 59.00 | - | 61.00 | 47.50 | 84.90 | 1288.9 | 59.60 |-|
|
59 |
| [MC-LLaVA-3b](https://huggingface.co/visheratin/MC-LLaVA-3b) | 3B | 64.24 | 49.60 | 24.88 | - | 38.59 | 80.59 | - | - |-|
|
60 |
-
| **Imp-
|
61 |
|
62 |
### Examples
|
|
|
63 |
![example1](images/example1.png)
|
64 |
|
65 |
## License
|
|
|
5 |
# 😈Imp
|
6 |
|
7 |
|
8 |
+
The Imp project aims to provide a family of a strong multimodal `small` language models (MSLMs). Our `imp-v1-3b` is a strong MSLM with only **3B** parameters, which is build upon a small yet powerful SLM [Phi-2 ](https://huggingface.co/microsoft/phi-2)(2.7B) and a powerful visual encoder [SigLIP ](https://huggingface.co/google/siglip-so400m-patch14-384)(0.4B), and trained on the [LLaVA-v1.5](https://github.com/haotian-liu/LLaVA) training set.
|
9 |
|
10 |
+
As shown in the Table below, `imp-v1-3b` significantly outperforms the counterparts of similar model sizes, and even achieves slightly better performance than the strong LLaVA-7B model on various multimodal benchmarks.
|
11 |
|
12 |
We release our model weights and provide an example below to run our model . Detailed technical report and corresponding training/evaluation code will be released soon on our [GitHub repo](https://github.com/MILVLG/imp). We will persistently improve our model and release the next versions to further improve model performance :)
|
13 |
|
|
|
25 |
|
26 |
#Create model
|
27 |
model = AutoModelForCausalLM.from_pretrained(
|
28 |
+
"MILVLG/imp-v1-3b",
|
29 |
torch_dtype=torch.float16,
|
30 |
device_map="auto",
|
31 |
trust_remote_code=True)
|
32 |
+
tokenizer = AutoTokenizer.from_pretrained("MILVLG/imp-v1-3b", trust_remote_code=True)
|
33 |
|
34 |
#Set inputs
|
35 |
text = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: <image>\nWhat are the colors of the bus in the image? ASSISTANT:"
|
|
|
57 |
| [LLaVA-Phi](https://github.com/zhuyiche/llava-phi) | 3B | 71.40 | - | 35.90 | 68.40 | 48.60 | 85.00 | 1335.1 | 59.80 |28.9|
|
58 |
| [MobileVLM](https://huggingface.co/mtgv/MobileVLM-3B) | 3B | - | 59.00 | - | 61.00 | 47.50 | 84.90 | 1288.9 | 59.60 |-|
|
59 |
| [MC-LLaVA-3b](https://huggingface.co/visheratin/MC-LLaVA-3b) | 3B | 64.24 | 49.60 | 24.88 | - | 38.59 | 80.59 | - | - |-|
|
60 |
+
| **Imp-v1 (ours)** | 3B | **79.45** | 58.55 | **50.09** |**69.96**| **59.38** | **88.02**| 1434.0 | **66.49** |**33.1**|
|
61 |
|
62 |
### Examples
|
63 |
+
|
64 |
![example1](images/example1.png)
|
65 |
|
66 |
## License
|
config.json
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
{
|
2 |
-
"_name_or_path": "MILVLG/Imp-
|
3 |
"activation_function": "gelu_new",
|
4 |
"architectures": [
|
5 |
"ImpForCausalLM"
|
|
|
1 |
{
|
2 |
+
"_name_or_path": "MILVLG/Imp-v1-3b",
|
3 |
"activation_function": "gelu_new",
|
4 |
"architectures": [
|
5 |
"ImpForCausalLM"
|
configuration_imp.py
CHANGED
@@ -176,7 +176,7 @@ class ImpConfig(PhiConfig):
|
|
176 |
@property
|
177 |
def vision_tower_cfg(self):
|
178 |
cfg = SiglipVisionConfig.from_dict(self.vision_tower_config)
|
179 |
-
# imp-
|
180 |
# cfg.mm_vision_select_feature = self.mm_vision_select_feature
|
181 |
cfg.mm_vision_select_layer = self.mm_vision_select_layer
|
182 |
cfg.mm_vision_tower = self.mm_vision_tower
|
|
|
176 |
@property
|
177 |
def vision_tower_cfg(self):
|
178 |
cfg = SiglipVisionConfig.from_dict(self.vision_tower_config)
|
179 |
+
# imp-v1 only supports `patch` feature for now w/o cls token
|
180 |
# cfg.mm_vision_select_feature = self.mm_vision_select_feature
|
181 |
cfg.mm_vision_select_layer = self.mm_vision_select_layer
|
182 |
cfg.mm_vision_tower = self.mm_vision_tower
|
requirements.txt
DELETED
@@ -1,4 +0,0 @@
|
|
1 |
-
torch>=2.0.1
|
2 |
-
transformers>=4.31.0
|
3 |
-
ftfy>=6.1.3
|
4 |
-
timm>=0.9.12
|
|
|
|
|
|
|
|
|
|