Hugging Face's logo

nvidia
/
NVLM-D-72B

Image-Text-to-Text
Transformers
Safetensors
PyTorch
English
NVLM_D
nvidia
NVLM
multimodal
conversational
custom_code
Inference Endpoints
Model card Files Community
29
NVLM-D-72B
File size: 6,351 Bytes 
b925209
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
datasets:
  coco_caption:
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\nGive a brief description of this image in one sentence.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    output_max_len: 30
    top_k: 3
    temperature: 1.0

  flickr30k_caption:
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\nGive a brief description of this image in one sentence.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    output_max_len: 30
    top_k: 3
    temperature: 1.0

  vqav2:
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}\nAnswer the question using a single word or phrase.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 8
    temperature: 1.0

  mmmu:
    split: "validation"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 1024
    temperature: 1.0
    apply_lemmatizer: False
    task_instructions: ""
    multi_choice_example_format: "{}\n{}\nAnswer with the option's letter from the given choices directly."
    short_ans_example_format: "{}\nAnswer the question using a single word or phrase."
    use_chat_format: True
    conv_format: "yi_nous_sft"
    default_image_token: "<image>"
    prompt_offset: 4
    answer_dict: "path/to/answer_dict_val.json"

  textvqa:
    split: "val"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}\nAnswer the question using a single word, phrase, or number.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 10
    temperature: 1.0

  mathvista:
    split: "testmini"
    prompt: "<|im_start|>system\nYou are math expert. Use your math knowledge to calculate the answer.<|im_end|><|im_start|>user\n<image>\n{}\nAnswer the question using a single word, phrase, or number.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 1024
    temperature: 1.0

  mmbench:
    split: "dev"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}Answer with the option's letter from the given choices directly.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 10
    temperature: 1.0
    submission: False

  chartqa:
    split: "test"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}<|im_end|><|im_start|>assistant\n"
    
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 20
    temperature: 1.0

  docvqa:
    split: "val"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 20
    temperature: 1.0

  realworldqa:
    split: "test"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 20
    temperature: 1.0
    submission: False

  ocrbench:
    split: "test"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 70
    temperature: 1.0
    submission: False

  ai2diagram:
    split: "test"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}\nAnswer the question using a single word, phrase, or number.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 20
    temperature: 1.0

  ai2diagram_nomask:
    split: "test"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|><|im_start|>user\n<image>\n{}\nAnswer the question using a single word, phrase, or number.<|im_end|><|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 20
    temperature: 1.0

  mmmu_pro:
    split: "validation"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 10
    temperature: 1.0
    apply_lemmatizer: False
    task_instructions: ""
    multi_choice_example_format: "{}\n{}\nAnswer with the option's letter from the given choices directly."
    short_ans_example_format: "{}\nAnswer the question using a single word or phrase."
    use_chat_format: True
    conv_format: "yi_nous_sft"
    default_image_token: "<image>"
    prompt_offset: 4
    answer_dict: "path/to/answer_dict.json"

  docvqa_test:
    split: "test"
    image_dir: "path/to/image"
    gt_path: "path/to/ground_truth"
    prompt: "<|im_start|>system\nFollow the user's instruction and answer questions.<|im_end|>\n<|im_start|>user\n<image>\n{}\nAnswer this question using the text in the image directly.<|im_end|>\n<|im_start|>assistant\n"
    beam_search: True
    beam_size: 1
    top_k: 1
    top_p: 0.0
    output_max_len: 20
    temperature: 1.0