HuggingFaceTB

/

SmolVLM2-2.2B-Instruct

like 50

Follow

Hugging Face TB Research 1.23k

Running the example of the model card page:

import torch
from transformers import AutoProcessor, AutoModelForImageTextToText

model_path = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
processor = AutoProcessor.from_pretrained(model_path)
model = AutoModelForImageTextToText.from_pretrained(
    model_path,
    torch_dtype=torch.bfloat16,
    _attn_implementation="flash_attention_2"
).to("cuda")

messages = [
    {
        "role": "user",
        "content": [
            {"type": "text", "text": "What is in this image?"},
            {"type": "image", "path": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"},

        ]
    },
]

inputs = processor.apply_chat_template(
    messages,
    add_generation_prompt=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to(model.device)

generated_ids = model.generate(**inputs, do_sample=False, max_new_tokens=64)
generated_texts = processor.batch_decode(
    generated_ids,
    skip_special_tokens=True,
)
print(generated_texts[0])

delivers the following error message

Traceback (most recent call last):
  File "C:\Lang\Python\SurveillanceVideo\smolVLM2.py", line 49, in <module>
    generated_ids = model.generate(**inputs, do_sample=False, max_new_tokens=64)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\utils\_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\transformers\generation\utils.py", line 2227, in generate
    result = self._sample(
        input_ids,
    ...<5 lines>...
        **model_kwargs,
    )
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\transformers\generation\utils.py", line 3215, in _sample
    outputs = self(**model_inputs, return_dict=True)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1739, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1750, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\transformers\models\smolvlm\modeling_smolvlm.py", line 1148, in forward
    outputs = self.model(
        input_ids=input_ids,
    ...<11 lines>...
        return_dict=return_dict,
    )
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1739, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1750, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\transformers\models\smolvlm\modeling_smolvlm.py", line 940, in forward
    image_hidden_states = self.vision_model(
                          ~~~~~~~~~~~~~~~~~^
        pixel_values=pixel_values,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^
        patch_attention_mask=patch_attention_mask,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ).last_hidden_state
    ^
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1739, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1750, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\transformers\models\smolvlm\modeling_smolvlm.py", line 564, in forward
    hidden_states = self.embeddings(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1739, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1750, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\transformers\models\smolvlm\modeling_smolvlm.py", line 140, in forward
    patch_embeds = self.patch_embedding(pixel_values)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1739, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\module.py", line 1750, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\conv.py", line 554, in forward
    return self._conv_forward(input, self.weight, self.bias)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Neiko\miniforge3\envs\SmolVLM2\Lib\site-packages\torch\nn\modules\conv.py", line 549, in _conv_forward
    return F.conv2d(
           ~~~~~~~~^
        input, weight, bias, self.stride, self.padding, self.dilation, self.groups
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (CUDABFloat16Type) should be the same