image_embedding_phi3_v.py

import warnings
import torch
from torch import nn
from transformers import CLIPVisionConfig, CLIPVisionModel, PretrainedConfig
from transformers.models.clip.modeling_clip import CLIPAttention
from transformers.utils import logging

try:
    from flash_attn import flash_attn_func
except ImportError:
    pass

logger = logging.get_logger(__name__)


MAX_INPUT_ID = int(1e9)

CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(
    attention_dropout=0.0,
    dropout=0.0,
    hidden_act="quick_gelu",
    hidden_size=1024,
    image_size=336,
    initializer_factor=1.0,
    initializer_range=0.02,
    intermediate_size=4096,
    layer_norm_eps=1e-05,
    num_attention_heads=16,
    num_channels=3,
    num_hidden_layers=24,
    patch_size=14,
    projection_dim=768
)


class CLIPAttentionFA2(CLIPAttention):
    """Add flash attention 2 to CLIPAttention. (This is only used in the vision encoder)"""

    def forward(self,
                hidden_states,
                attention_mask=None,
                causal_attention_mask=None,
                output_attentions=False):
        """Input shape: Batch x Time x Channel"""
        assert attention_mask is None, "CLIPAttentionFA2 does not support attention_mask"
        assert causal_attention_mask is None, "CLIPAttentionFA2 does not support causal_attention_mask"
        assert output_attentions is False, "CLIPAttentionFA2 does not support output_attentions"

        bsz, tgt_len, embed_dim = hidden_states.size()
        query_states = self.q_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim)
        key_states = self.k_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim)
        value_states = self.v_proj(hidden_states).reshape(bsz, tgt_len, self.num_heads, self.head_dim)

        attn_output = flash_attn_func(
            query_states,
            key_states,
            value_states,
            dropout_p=self.dropout if self.training else 0.0,
            softmax_scale=self.scale,
            causal=False,
        ).reshape(bsz, tgt_len, embed_dim)

        attn_output = self.out_proj(attn_output)
        return attn_output, None


class Phi3ImageEmbedding(nn.Module):
    """Phi3 Image embedding with support for batched multi-image input."""

    def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None:
        super().__init__()

        # n_embed or hidden_size
        hidden_size = config.n_embd if hasattr(config, 'n_embd') else config.hidden_size
        if hasattr(config, 'embd_pdrop') or hasattr(config, 'embed_pdrop'):
            embd_drop = config.embd_pdrop if hasattr(config, 'embd_pdrop') else config.embed_pdrop
            self.drop = nn.Dropout(embd_drop)
        else:
            self.drop = None

        self.wte = wte

        if isinstance(config.img_processor, dict) and config.img_processor.get('name', None) == 'clip_vision_model':
            assert 'model_name' in config.img_processor, 'model_name must be provided for CLIPVisionModel'
            assert 'image_dim_out' in config.img_processor, 'image_dim_out must be provided for CLIPVisionModel'
            assert 'num_img_tokens' in config.img_processor, 'num_img_tokens must be provided for CLIPVisionModel'
            assert config.img_processor['model_name'] == 'openai/clip-vit-large-patch14-336'
            clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
            self.img_processor = CLIPVisionModel(clip_config)
            image_dim_out = config.img_processor['image_dim_out']
            self.num_img_tokens = config.img_processor['num_img_tokens']

            # FA2 in CLIP
            if config._attn_implementation == 'flash_attention_2':
                for layer in self.img_processor.vision_model.encoder.layers:
                    clip_fa2 = CLIPAttentionFA2(clip_config)
                    del layer.self_attn
                    layer.self_attn = clip_fa2
        else:
            raise NotImplementedError(f'img_processor = {config.img_processor}, not implemented')

        self.image_dim_out = image_dim_out
        self.img_sizes = None

        # global_gn and sub_gn for hd transform, serves as line separator
        self.use_hd_transform = kwargs.get('use_hd_transform', False)
        self.with_learnable_separator = kwargs.get('with_learnable_separator', False)
        self.hd_transform_order = kwargs.get('hd_transform_order', 'glb_sub')
        assert self.use_hd_transform == self.with_learnable_separator, 'use_hd_transform and with_learnable_separator should have same value'

        if self.with_learnable_separator:
            assert self.use_hd_transform, 'learnable separator is only for hd transform'
            # 1024 * 4, merge spatial to channel dimension
            self.glb_GN = nn.Parameter(torch.zeros([1, 1, self.image_dim_out * 4]))
            self.sub_GN = nn.Parameter(torch.zeros([1, 1, 1, self.image_dim_out * 4]))
            logger.info(f'learnable separator enabled for hd transform, hd_transform_order = {self.hd_transform_order}')

        projection_cls = kwargs.get('projection_cls', 'linear')
        if projection_cls == 'linear':
            self.img_projection = nn.Linear(image_dim_out, hidden_size)
        elif projection_cls == 'mlp' and self.use_hd_transform:
            dim_projection = hidden_size
            depth = 2
            layers = [nn.Linear(image_dim_out * 4, dim_projection)]
            for _ in range(1, depth):
                layers.extend([nn.GELU(),
                               nn.Linear(dim_projection, dim_projection)])
            self.img_projection = nn.Sequential(*layers)
        elif projection_cls == 'mlp':
            dim_projection = hidden_size
            depth = 2
            layers = [nn.Linear(image_dim_out, dim_projection)]
            for _ in range(1, depth):
                layers.extend([nn.GELU(),
                               nn.Linear(dim_projection, dim_projection)])
            self.img_projection = nn.Sequential(*layers)
        else:
            raise NotImplementedError(f'projection_cls = {projection_cls}, not implemented')

        self.vocab_size = config.vocab_size
        self.img_features = None

        if isinstance(config.img_processor, dict):
            self.layer_idx = config.img_processor.get('layer_idx', -2)
            self.type_feature = config.img_processor.get('type_feature', 'patch')
        else:
            self.layer_idx = -2
            self.type_feature = 'patch'

    def set_img_features(self, img_features: torch.FloatTensor) -> None:
        self.img_features = img_features

    def set_img_sizes(self, img_sizes: torch.LongTensor) -> None:
        self.img_sizes = img_sizes

    def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor:
        """
        img_embeds: (N, 3, 336, 336)
        Returns: (N, L, C) where L=24*24=576 and C=1024
        """
        LAYER_IDX = self.layer_idx
        TYPE_FEATURE = self.type_feature

        img_processor_output = self.img_processor(img_embeds, output_hidden_states=True)
        img_feature = img_processor_output.hidden_states[LAYER_IDX]

        if TYPE_FEATURE == "patch":
            patch_feature = img_feature[:, 1:]
            return patch_feature

        raise NotImplementedError

    def forward(
        self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, image_sizes=None
    ) -> torch.FloatTensor:
        """
        pixel_values: (batch_size, num_images, num_crops+1, 3, 336, 336)
        image_sizes: (batch_size, num_images, 2)
        """
        input_shape = input_ids.size()
        input_ids = input_ids.view(-1, input_shape[-1])

        # positions for image tokens
        positions = torch.nonzero((input_ids < 0) & (input_ids > -MAX_INPUT_ID), as_tuple=True)
        has_image = len(positions[0].tolist()) > 0

        # clamp input_ids
        input_ids.clamp_min_(0).clamp_max_(self.vocab_size)
        warnings.warn(
            "Phi-3-V modifies `input_ids` in-place and the tokens indicating images will be "
            "removed after model forward. If your workflow requires multiple forward passes on "
            "the same `input_ids`, please make a copy of `input_ids` before passing it to the "
            "model."
        )

        hidden_states = self.wte(input_ids)

        if has_image:
            assert self.use_hd_transform, "Image insertion requires HD transform enabled."
            b, n, m, c, h, w = pixel_values.shape  # b= batch_size, n= num_images, m= num_crops+1
            assert c == 3 and h == w == 336

            # Flatten batch and image dimension for feature extraction
            pixel_values_flat = pixel_values.reshape(b * n * m, c, h, w)
            img_features = self.get_img_features(pixel_values_flat)  # (b*n*m, 576, 1024)
            img_features = img_features.reshape(b * n, m, 576, self.image_dim_out)

            # Flatten image_sizes as well
            image_sizes_flat = image_sizes.reshape(b * n, 2)

            image_features_proj = self.hd_feature_transform(img_features, image_sizes_flat)
            hidden_states = hidden_states.index_put(
                positions, image_features_proj, accumulate=False
            )

        if self.drop is not None:
            hidden_states = self.drop(hidden_states)

        return hidden_states

    def hd_feature_transform(self, image_features, image_sizes):
        """
        HD transform over multiple images.

        image_features: (N, m, 576, 1024)
            N = b*n (total number of images in the batch)
            m = num_crops+1
            The first crop is global, the remaining are sub-crops.

        image_sizes: (N, 2)
            image_sizes[i] = (height, width) for the i-th image.

        Returns:
            image_features_proj: Concatenated image embeddings projected to hidden size.
        """
        assert self.hd_transform_order == 'sub_glb', f'hd_transform_order `{self.hd_transform_order}` not implemented'

        if isinstance(self.img_projection, nn.Sequential):
            target_device = self.img_projection[0].bias.device
            target_dtype = self.img_projection[0].bias.dtype
        else:  # single nn.Linear layer
            target_device = self.img_projection.bias.device
            target_dtype = self.img_projection.bias.dtype

        N, m, L, C = image_features.shape  # L=576=24*24, C=1024
        num_crops = m - 1

        # global_image_features: (N, 576, 1024)
        global_image_features = image_features[:, 0]

        # process global features into HD format
        global_image_features_hd = self.reshape_hd_patches_2x2merge(global_image_features, 1, 1)
        global_image_features_hd_newline = self.add_image_newline(global_image_features_hd)

        all_image_embeddings = []
        for i in range(N):
            h, w = image_sizes[i]
            h_crop = h // 336
            w_crop = w // 336

            sub_image_features = image_features[i, 1:1 + num_crops]  # (num_crops, 576, 1024)
            sub_image_features_hd = self.reshape_hd_patches_2x2merge(sub_image_features, h_crop, w_crop)
            sub_image_features_hd_newline = self.add_image_newline(sub_image_features_hd)

            # [sub features, separator, global features]
            all_image_embeddings.extend([
                sub_image_features_hd_newline.squeeze(0),
                self.glb_GN.squeeze(0),
                global_image_features_hd_newline[i],
            ])

        image_features_proj = self.img_projection(
            torch.cat(all_image_embeddings, dim=0).to(target_device).to(target_dtype)
        )

        return image_features_proj

    def reshape_hd_patches_2x2merge(self, image_features, h_crop, w_crop):
        """
        image_features: (N, L, C) or (N, 24*24, 1024)
        Reshape into HD patches:
        output: (N_img, h_crop*12, w_crop*12, 4096)
        """
        N, L, C = image_features.shape
        assert L == 24 * 24 and C == 1024
        H = int(L**0.5)
        num_images = N // (h_crop * w_crop) if N != 1 else 1

        image_features_hd = (
            image_features.reshape(N, H, H, C)  # N, 24, 24, 1024
            .reshape(N, H // 2, 2, H // 2, 2, C)  # N, 12, 2, 12, 2, 1024
            .permute(0, 1, 3, 2, 4, 5)  # N, 12, 12, 2, 2, 1024
            .reshape(N, -1, 4 * C)  # N, 144, 4096
        )

        # Now group them by images if N = num_images * num_crops:
        # shape: (num_images, h_crop, w_crop, 12, 12, 4096)
        image_features_hd = image_features_hd.reshape(num_images, h_crop, w_crop, H // 2, H // 2, 4 * C)
        # rearrange to (num_images, h_crop*12, w_crop*12, 4096)
        image_features_hd = image_features_hd.permute(0, 1, 3, 2, 4, 5).reshape(
            num_images, h_crop * (H // 2), w_crop * (H // 2), 4 * C
        )

        return image_features_hd

    def add_image_newline(self, image_features_hd):
        """
        image_features_hd: (num_images, h, w, 4096)
        Add newline token: (num_images, h*(w+1), 4096)
        """
        num_images, h, w, hid_dim = image_features_hd.shape
        newline_embeddings = self.sub_GN.expand(num_images, h, 1, hid_dim)
        image_features_hd_newline = torch.cat(
            [image_features_hd, newline_embeddings], dim=2
        ).reshape(num_images, -1, hid_dim)
        return image_features_hd_newline