Delete extras

extras/BLIP/configs/bert_config.json

@@ -1,21 +0,0 @@
-{
-  "architectures": [
-    "BertModel"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "type_vocab_size": 2,
-  "vocab_size": 30522,
-  "encoder_width": 768,
-  "add_cross_attention": true   
-}

extras/BLIP/configs/caption_coco.yaml

@@ -1,33 +0,0 @@
-image_root: '/export/share/datasets/vision/coco/images/'
-ann_root: 'annotation'
-coco_gt_root: 'annotation/coco_gt'
-
-# set pretrained as a file path or an url
-pretrained: 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth'
-
-# size of vit model; base or large
-vit: 'base'
-vit_grad_ckpt: False
-vit_ckpt_layer: 0
-batch_size: 32
-init_lr: 1e-5
-
-# vit: 'large'
-# vit_grad_ckpt: True
-# vit_ckpt_layer: 5
-# batch_size: 16
-# init_lr: 2e-6
-
-image_size: 384
-
-# generation configs
-max_length: 20  
-min_length: 5
-num_beams: 3
-prompt: 'a picture of '
-
-# optimizer
-weight_decay: 0.05
-min_lr: 0
-max_epoch: 5
-

extras/BLIP/configs/med_config.json

@@ -1,21 +0,0 @@
-{
-  "architectures": [
-    "BertModel"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "type_vocab_size": 2,
-  "vocab_size": 30524,
-  "encoder_width": 768,
-  "add_cross_attention": true   
-}

extras/BLIP/configs/nlvr.yaml

@@ -1,21 +0,0 @@
-image_root: '/export/share/datasets/vision/NLVR2/' 
-ann_root: 'annotation'
-
-# set pretrained as a file path or an url
-pretrained: 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_nlvr.pth'
-
-#size of vit model; base or large
-vit: 'base'
-batch_size_train: 16 
-batch_size_test: 64 
-vit_grad_ckpt: False
-vit_ckpt_layer: 0
-max_epoch: 15
-
-image_size: 384
-
-# optimizer
-weight_decay: 0.05
-init_lr: 3e-5
-min_lr: 0
-

extras/BLIP/configs/nocaps.yaml

@@ -1,15 +0,0 @@
-image_root: '/export/share/datasets/vision/nocaps/'
-ann_root: 'annotation'
-
-# set pretrained as a file path or an url
-pretrained: 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth'
-
-vit: 'base'
-batch_size: 32
-
-image_size: 384
-
-max_length: 20
-min_length: 5
-num_beams: 3
-prompt: 'a picture of '

extras/BLIP/configs/pretrain.yaml

@@ -1,27 +0,0 @@
-train_file: ['/export/share/junnan-li/VL_pretrain/annotation/coco_karpathy_train.json',
-             '/export/share/junnan-li/VL_pretrain/annotation/vg_caption.json',
-             ]
-laion_path: ''   
-
-# size of vit model; base or large
-vit: 'base'
-vit_grad_ckpt: False
-vit_ckpt_layer: 0
-
-image_size: 224
-batch_size: 75
-
-queue_size: 57600
-alpha: 0.4
-
-# optimizer
-weight_decay: 0.05
-init_lr: 3e-4
-min_lr: 1e-6
-warmup_lr: 1e-6
-lr_decay_rate: 0.9
-max_epoch: 20
-warmup_steps: 3000
-
-
-

extras/BLIP/configs/retrieval_coco.yaml

@@ -1,34 +0,0 @@
-image_root: '/export/share/datasets/vision/coco/images/'
-ann_root: 'annotation'
-dataset: 'coco'
-
-# set pretrained as a file path or an url
-pretrained: 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth'
-
-# size of vit model; base or large
-
-vit: 'base'
-batch_size_train: 32
-batch_size_test: 64
-vit_grad_ckpt: True
-vit_ckpt_layer: 4
-init_lr: 1e-5
-
-# vit: 'large'
-# batch_size_train: 16
-# batch_size_test: 32
-# vit_grad_ckpt: True
-# vit_ckpt_layer: 12
-# init_lr: 5e-6
-
-image_size: 384
-queue_size: 57600
-alpha: 0.4
-k_test: 256
-negative_all_rank: True
-
-# optimizer
-weight_decay: 0.05
-min_lr: 0
-max_epoch: 6
-

extras/BLIP/configs/retrieval_flickr.yaml

@@ -1,34 +0,0 @@
-image_root: '/export/share/datasets/vision/flickr30k/'
-ann_root: 'annotation'
-dataset: 'flickr'
-
-# set pretrained as a file path or an url
-pretrained: 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_flickr.pth'
-
-# size of vit model; base or large
-
-vit: 'base'
-batch_size_train: 32
-batch_size_test: 64
-vit_grad_ckpt: True
-vit_ckpt_layer: 4
-init_lr: 1e-5
-
-# vit: 'large'
-# batch_size_train: 16
-# batch_size_test: 32
-# vit_grad_ckpt: True
-# vit_ckpt_layer: 10
-# init_lr: 5e-6
-
-image_size: 384
-queue_size: 57600
-alpha: 0.4
-k_test: 128
-negative_all_rank: False
-
-# optimizer
-weight_decay: 0.05
-min_lr: 0
-max_epoch: 6
-

extras/BLIP/configs/retrieval_msrvtt.yaml

@@ -1,12 +0,0 @@
-video_root: '/export/share/dongxuli/data/msrvtt_retrieval/videos'
-ann_root: 'annotation'
-
-# set pretrained as a file path or an url
-pretrained: 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth'
-
-# size of vit model; base or large
-vit: 'base'
-batch_size: 64
-k_test: 128
-image_size: 384
-num_frm_test: 8

extras/BLIP/configs/vqa.yaml

@@ -1,25 +0,0 @@
-vqa_root: '/export/share/datasets/vision/VQA/Images/mscoco/' #followed by train2014/
-vg_root: '/export/share/datasets/vision/visual-genome/'  #followed by image/
-train_files: ['vqa_train','vqa_val','vg_qa']
-ann_root: 'annotation'
-
-# set pretrained as a file path or an url
-pretrained: 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_vqa_capfilt_large.pth'
-
-# size of vit model; base or large
-vit: 'base'
-batch_size_train: 16 
-batch_size_test: 32 
-vit_grad_ckpt: False
-vit_ckpt_layer: 0
-init_lr: 2e-5
-
-image_size: 480
-
-k_test: 128
-inference: 'rank'
-
-# optimizer
-weight_decay: 0.05
-min_lr: 0
-max_epoch: 10

extras/BLIP/models/bert_tokenizer/config.json

@@ -1,23 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "gradient_checkpointing": false,
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "transformers_version": "4.6.0.dev0",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 30522
-}

extras/BLIP/models/bert_tokenizer/tokenizer_config.json

@@ -1,3 +0,0 @@
-{
-  "do_lower_case": true
-}

extras/BLIP/models/blip.py

@@ -1,239 +0,0 @@
-'''
- * Copyright (c) 2022, salesforce.com, inc.
- * All rights reserved.
- * SPDX-License-Identifier: BSD-3-Clause
- * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
- * By Junnan Li
-'''
-import warnings
-warnings.filterwarnings("ignore")
-
-from extras.BLIP.models.vit import VisionTransformer, interpolate_pos_embed
-from extras.BLIP.models.med import BertConfig, BertModel, BertLMHeadModel
-from transformers import BertTokenizer
-
-import torch
-from torch import nn
-import torch.nn.functional as F
-
-import os
-from urllib.parse import urlparse
-from timm.models.hub import download_cached_file
-
-class BLIP_Base(nn.Module):
-    def __init__(self,                 
-                 med_config = 'configs/med_config.json',  
-                 image_size = 224,
-                 vit = 'base',
-                 vit_grad_ckpt = False,
-                 vit_ckpt_layer = 0,                 
-                 ):
-        """
-        Args:
-            med_config (str): path for the mixture of encoder-decoder model's configuration file
-            image_size (int): input image size
-            vit (str): model size of vision transformer
-        """               
-        super().__init__()
-        
-        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
-        self.tokenizer = init_tokenizer()   
-        med_config = BertConfig.from_json_file(med_config)
-        med_config.encoder_width = vision_width
-        self.text_encoder = BertModel(config=med_config, add_pooling_layer=False)  
-
-        
-    def forward(self, image, caption, mode):
-        
-        assert mode in ['image', 'text', 'multimodal'], "mode parameter must be image, text, or multimodal"
-        text = self.tokenizer(caption, return_tensors="pt").to(image.device) 
-        
-        if mode=='image':    
-            # return image features
-            image_embeds = self.visual_encoder(image)             
-            return image_embeds
-        
-        elif mode=='text':
-            # return text features
-            text_output = self.text_encoder(text.input_ids, attention_mask = text.attention_mask,                      
-                                            return_dict = True, mode = 'text')  
-            return text_output.last_hidden_state
-        
-        elif mode=='multimodal':
-            # return multimodel features
-            image_embeds = self.visual_encoder(image)    
-            image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)      
-            
-            text.input_ids[:,0] = self.tokenizer.enc_token_id
-            output = self.text_encoder(text.input_ids,
-                                       attention_mask = text.attention_mask,
-                                       encoder_hidden_states = image_embeds,
-                                       encoder_attention_mask = image_atts,      
-                                       return_dict = True,
-                                      )              
-            return output.last_hidden_state
-        
-        
-        
-class BLIP_Decoder(nn.Module):
-    def __init__(self,                 
-                 med_config = 'configs/med_config.json',  
-                 image_size = 384,
-                 vit = 'base',
-                 vit_grad_ckpt = False,
-                 vit_ckpt_layer = 0,
-                 prompt = 'a picture of ',
-                 ):
-        """
-        Args:
-            med_config (str): path for the mixture of encoder-decoder model's configuration file
-            image_size (int): input image size
-            vit (str): model size of vision transformer
-        """            
-        super().__init__()
-        
-        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
-        self.tokenizer = init_tokenizer()   
-        med_config = BertConfig.from_json_file(med_config)
-        med_config.encoder_width = vision_width
-        self.text_decoder = BertLMHeadModel(config=med_config)    
-        
-        self.prompt = prompt
-        self.prompt_length = len(self.tokenizer(self.prompt).input_ids)-1
-
-        
-    def forward(self, image, caption):
-        
-        image_embeds = self.visual_encoder(image) 
-        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
-        
-        text = self.tokenizer(caption, padding='longest', truncation=True, max_length=40, return_tensors="pt").to(image.device) 
-        
-        text.input_ids[:,0] = self.tokenizer.bos_token_id
-        
-        decoder_targets = text.input_ids.masked_fill(text.input_ids == self.tokenizer.pad_token_id, -100)         
-        decoder_targets[:,:self.prompt_length] = -100
-     
-        decoder_output = self.text_decoder(text.input_ids, 
-                                           attention_mask = text.attention_mask, 
-                                           encoder_hidden_states = image_embeds,
-                                           encoder_attention_mask = image_atts,                  
-                                           labels = decoder_targets,
-                                           return_dict = True,   
-                                          )   
-        loss_lm = decoder_output.loss
-        
-        return loss_lm
-        
-    def generate(self, image, sample=False, num_beams=3, max_length=30, min_length=10, top_p=0.9, repetition_penalty=1.0):
-        image_embeds = self.visual_encoder(image)
-
-        if not sample:
-            image_embeds = image_embeds.repeat_interleave(num_beams,dim=0)
-            
-        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
-        model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask":image_atts}
-        
-        prompt = [self.prompt] * image.size(0)
-        input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids.to(image.device) 
-        input_ids[:,0] = self.tokenizer.bos_token_id
-        input_ids = input_ids[:, :-1] 
-
-        if sample:
-            #nucleus sampling
-            outputs = self.text_decoder.generate(input_ids=input_ids,
-                                                  max_length=max_length,
-                                                  min_length=min_length,
-                                                  do_sample=True,
-                                                  top_p=top_p,
-                                                  num_return_sequences=1,
-                                                  eos_token_id=self.tokenizer.sep_token_id,
-                                                  pad_token_id=self.tokenizer.pad_token_id, 
-                                                  repetition_penalty=1.1,                                            
-                                                  **model_kwargs)
-        else:
-            #beam search
-            outputs = self.text_decoder.generate(input_ids=input_ids,
-                                                  max_length=max_length,
-                                                  min_length=min_length,
-                                                  num_beams=num_beams,
-                                                  eos_token_id=self.tokenizer.sep_token_id,
-                                                  pad_token_id=self.tokenizer.pad_token_id,     
-                                                  repetition_penalty=repetition_penalty,
-                                                  **model_kwargs)            
-            
-        captions = []    
-        for output in outputs:
-            caption = self.tokenizer.decode(output, skip_special_tokens=True)    
-            captions.append(caption[len(self.prompt):])
-        return captions
-    
-
-def blip_decoder(pretrained='',**kwargs):
-    model = BLIP_Decoder(**kwargs)
-    if pretrained:
-        model,msg = load_checkpoint(model,pretrained)
-        assert(len(msg.missing_keys)==0)
-    return model    
-    
-def blip_feature_extractor(pretrained='',**kwargs):
-    model = BLIP_Base(**kwargs)
-    if pretrained:
-        model,msg = load_checkpoint(model,pretrained)
-        assert(len(msg.missing_keys)==0)
-    return model        
-
-def init_tokenizer():
-    tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bert_tokenizer")
-    tokenizer = BertTokenizer.from_pretrained(tokenizer_path)
-    tokenizer.add_special_tokens({'bos_token':'[DEC]'})
-    tokenizer.add_special_tokens({'additional_special_tokens':['[ENC]']})       
-    tokenizer.enc_token_id = tokenizer.additional_special_tokens_ids[0]  
-    return tokenizer
-
-
-def create_vit(vit, image_size, use_grad_checkpointing=False, ckpt_layer=0, drop_path_rate=0):
-        
-    assert vit in ['base', 'large'], "vit parameter must be base or large"
-    if vit=='base':
-        vision_width = 768
-        visual_encoder = VisionTransformer(img_size=image_size, patch_size=16, embed_dim=vision_width, depth=12, 
-                                           num_heads=12, use_grad_checkpointing=use_grad_checkpointing, ckpt_layer=ckpt_layer,
-                                           drop_path_rate=0 or drop_path_rate
-                                          )   
-    elif vit=='large':
-        vision_width = 1024
-        visual_encoder = VisionTransformer(img_size=image_size, patch_size=16, embed_dim=vision_width, depth=24, 
-                                           num_heads=16, use_grad_checkpointing=use_grad_checkpointing, ckpt_layer=ckpt_layer,
-                                           drop_path_rate=0.1 or drop_path_rate
-                                          )   
-    return visual_encoder, vision_width
-
-def is_url(url_or_filename):
-    parsed = urlparse(url_or_filename)
-    return parsed.scheme in ("http", "https")
-
-def load_checkpoint(model,url_or_filename):
-    if is_url(url_or_filename):
-        cached_file = download_cached_file(url_or_filename, check_hash=False, progress=True)
-        checkpoint = torch.load(cached_file, map_location='cpu') 
-    elif os.path.isfile(url_or_filename):        
-        checkpoint = torch.load(url_or_filename, map_location='cpu') 
-    else:
-        raise RuntimeError('checkpoint url or path is invalid')
-        
-    state_dict = checkpoint['model']
-    
-    state_dict['visual_encoder.pos_embed'] = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) 
-    if 'visual_encoder_m.pos_embed' in model.state_dict().keys():
-        state_dict['visual_encoder_m.pos_embed'] = interpolate_pos_embed(state_dict['visual_encoder_m.pos_embed'],
-                                                                         model.visual_encoder_m)    
-    for key in model.state_dict().keys():
-        if key in state_dict.keys():
-            if state_dict[key].shape!=model.state_dict()[key].shape:
-                del state_dict[key]
-    
-    msg = model.load_state_dict(state_dict,strict=False)
-    print('load checkpoint from %s'%url_or_filename)  
-    return model,msg
-    

extras/BLIP/models/blip_itm.py

@@ -1,76 +0,0 @@
-from extras.BLIP.models.med import BertConfig, BertModel
-from transformers import BertTokenizer
-
-import torch
-from torch import nn
-import torch.nn.functional as F
-
-from extras.BLIP.models.blip import create_vit, init_tokenizer, load_checkpoint
-
-class BLIP_ITM(nn.Module):
-    def __init__(self,                 
-                 med_config = 'configs/med_config.json',  
-                 image_size = 384,
-                 vit = 'base',
-                 vit_grad_ckpt = False,
-                 vit_ckpt_layer = 0,                      
-                 embed_dim = 256,     
-                 ):
-        """
-        Args:
-            med_config (str): path for the mixture of encoder-decoder model's configuration file
-            image_size (int): input image size
-            vit (str): model size of vision transformer
-        """               
-        super().__init__()
-        
-        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
-        self.tokenizer = init_tokenizer()   
-        med_config = BertConfig.from_json_file(med_config)
-        med_config.encoder_width = vision_width
-        self.text_encoder = BertModel(config=med_config, add_pooling_layer=False)          
-
-        text_width = self.text_encoder.config.hidden_size
-        
-        self.vision_proj = nn.Linear(vision_width, embed_dim)
-        self.text_proj = nn.Linear(text_width, embed_dim)
-
-        self.itm_head = nn.Linear(text_width, 2) 
-        
-        
-    def forward(self, image, caption, match_head='itm'):
-
-        image_embeds = self.visual_encoder(image) 
-        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)        
-      
-        text = self.tokenizer(caption, padding='max_length', truncation=True, max_length=35, 
-                              return_tensors="pt").to(image.device) 
-
-                 
-        if match_head=='itm':
-            output = self.text_encoder(text.input_ids,
-                                       attention_mask = text.attention_mask,
-                                       encoder_hidden_states = image_embeds,
-                                       encoder_attention_mask = image_atts,      
-                                       return_dict = True,
-                                      )
-            itm_output = self.itm_head(output.last_hidden_state[:,0,:])     
-            return itm_output
-            
-        elif match_head=='itc':
-            text_output = self.text_encoder(text.input_ids, attention_mask = text.attention_mask,                      
-                                            return_dict = True, mode = 'text')                     
-            image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1)   
-            text_feat = F.normalize(self.text_proj(text_output.last_hidden_state[:,0,:]),dim=-1)    
-            
-            sim = image_feat @ text_feat.t()
-            return sim
-        
-        
-def blip_itm(pretrained='',**kwargs):
-    model = BLIP_ITM(**kwargs)
-    if pretrained:
-        model,msg = load_checkpoint(model,pretrained)
-        assert(len(msg.missing_keys)==0)
-    return model         
-            

extras/BLIP/models/blip_nlvr.py

@@ -1,105 +0,0 @@
-from extras.BLIP.models.med import BertConfig
-from extras.BLIP.models.nlvr_encoder import BertModel
-from extras.BLIP.models.vit import interpolate_pos_embed
-from extras.BLIP.models.blip import create_vit, init_tokenizer, is_url
-
-from timm.models.hub import download_cached_file
-
-import torch
-from torch import nn
-import torch.nn.functional as F
-from transformers import BertTokenizer
-import numpy as np
-import os
-
-
-class BLIP_NLVR(nn.Module):
-    def __init__(self,                 
-                 med_config = 'configs/med_config.json',  
-                 image_size = 480,
-                 vit = 'base',
-                 vit_grad_ckpt = False,
-                 vit_ckpt_layer = 0,                   
-                 ):
-        """
-        Args:
-            med_config (str): path for the mixture of encoder-decoder model's configuration file
-            image_size (int): input image size
-            vit (str): model size of vision transformer
-        """               
-        super().__init__()
-        
-        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer, drop_path_rate=0.1)
-        self.tokenizer = init_tokenizer()   
-        med_config = BertConfig.from_json_file(med_config)
-        med_config.encoder_width = vision_width
-        self.text_encoder = BertModel(config=med_config, add_pooling_layer=False) 
-                    
-        self.cls_head = nn.Sequential(
-                  nn.Linear(self.text_encoder.config.hidden_size, self.text_encoder.config.hidden_size),
-                  nn.ReLU(),
-                  nn.Linear(self.text_encoder.config.hidden_size, 2)
-                )  
-
-    def forward(self, image, text, targets, train=True):
-        
-        image_embeds = self.visual_encoder(image) 
-        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)        
-        image0_embeds, image1_embeds = torch.split(image_embeds,targets.size(0))     
-
-        text = self.tokenizer(text, padding='longest', return_tensors="pt").to(image.device) 
-        text.input_ids[:,0] = self.tokenizer.enc_token_id        
-
-        output = self.text_encoder(text.input_ids, 
-                                   attention_mask = text.attention_mask, 
-                                   encoder_hidden_states = [image0_embeds,image1_embeds],
-                                   encoder_attention_mask = [image_atts[:image0_embeds.size(0)],
-                                                             image_atts[image0_embeds.size(0):]],        
-                                   return_dict = True,
-                                  )  
-        hidden_state = output.last_hidden_state[:,0,:]        
-        prediction = self.cls_head(hidden_state)
-
-        if train:            
-            loss = F.cross_entropy(prediction, targets)   
-            return loss
-        else:
-            return prediction
-    
-def blip_nlvr(pretrained='',**kwargs):
-    model = BLIP_NLVR(**kwargs)
-    if pretrained:
-        model,msg = load_checkpoint(model,pretrained)
-        print("missing keys:")
-        print(msg.missing_keys)
-    return model  
-
-        
-def load_checkpoint(model,url_or_filename):
-    if is_url(url_or_filename):
-        cached_file = download_cached_file(url_or_filename, check_hash=False, progress=True)
-        checkpoint = torch.load(cached_file, map_location='cpu') 
-    elif os.path.isfile(url_or_filename):        
-        checkpoint = torch.load(url_or_filename, map_location='cpu') 
-    else:
-        raise RuntimeError('checkpoint url or path is invalid')
-    state_dict = checkpoint['model']
-    
-    state_dict['visual_encoder.pos_embed'] = interpolate_pos_embed(state_dict['visual_encoder.pos_embed'],model.visual_encoder) 
-    
-    for key in list(state_dict.keys()):
-        if 'crossattention.self.' in key:
-            new_key0 = key.replace('self','self0')
-            new_key1 = key.replace('self','self1')
-            state_dict[new_key0] = state_dict[key]
-            state_dict[new_key1] = state_dict[key]
-        elif 'crossattention.output.dense.' in key:
-            new_key0 = key.replace('dense','dense0')
-            new_key1 = key.replace('dense','dense1')
-            state_dict[new_key0] = state_dict[key]
-            state_dict[new_key1] = state_dict[key]  
-                
-    msg = model.load_state_dict(state_dict,strict=False)
-    print('load checkpoint from %s'%url_or_filename)  
-    return model,msg
-            

extras/BLIP/models/blip_pretrain.py

@@ -1,339 +0,0 @@
-'''
- * Copyright (c) 2022, salesforce.com, inc.
- * All rights reserved.
- * SPDX-License-Identifier: BSD-3-Clause
- * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
- * By Junnan Li
-'''
-from extras.BLIP.models.med import BertConfig, BertModel, BertLMHeadModel
-from transformers import BertTokenizer
-import transformers
-transformers.logging.set_verbosity_error()
-
-import torch
-from torch import nn
-import torch.nn.functional as F
-
-from extras.BLIP.models.blip import create_vit, init_tokenizer, load_checkpoint
-
-class BLIP_Pretrain(nn.Module):
-    def __init__(self,                 
-                 med_config = 'configs/bert_config.json',  
-                 image_size = 224,
-                 vit = 'base',
-                 vit_grad_ckpt = False,
-                 vit_ckpt_layer = 0,                    
-                 embed_dim = 256,     
-                 queue_size = 57600,
-                 momentum = 0.995,
-                 ):
-        """
-        Args:
-            med_config (str): path for the mixture of encoder-decoder model's configuration file
-            image_size (int): input image size
-            vit (str): model size of vision transformer
-        """               
-        super().__init__()
-        
-        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer, 0)
-        
-        if vit=='base':
-            checkpoint = torch.hub.load_state_dict_from_url(
-                url="https://dl.fbaipublicfiles.com/deit/deit_base_patch16_224-b5f2ef4d.pth",
-                map_location="cpu", check_hash=True)
-            state_dict = checkpoint["model"]     
-            msg = self.visual_encoder.load_state_dict(state_dict,strict=False)
-        elif vit=='large':
-            from timm.models.helpers import load_custom_pretrained
-            from timm.models.vision_transformer import default_cfgs
-            load_custom_pretrained(self.visual_encoder,default_cfgs['vit_large_patch16_224_in21k'])        
-               
-        self.tokenizer = init_tokenizer()   
-        encoder_config = BertConfig.from_json_file(med_config)
-        encoder_config.encoder_width = vision_width
-        self.text_encoder = BertModel.from_pretrained('bert-base-uncased',config=encoder_config, add_pooling_layer=False)
-        self.text_encoder.resize_token_embeddings(len(self.tokenizer)) 
-
-        text_width = self.text_encoder.config.hidden_size
-        
-        self.vision_proj = nn.Linear(vision_width, embed_dim)
-        self.text_proj = nn.Linear(text_width, embed_dim)
-
-        self.itm_head = nn.Linear(text_width, 2) 
-        
-        # create momentum encoders  
-        self.visual_encoder_m, vision_width = create_vit(vit,image_size)              
-        self.vision_proj_m = nn.Linear(vision_width, embed_dim)
-        self.text_encoder_m = BertModel(config=encoder_config, add_pooling_layer=False)      
-        self.text_proj_m = nn.Linear(text_width, embed_dim)
-        
-        self.model_pairs = [[self.visual_encoder,self.visual_encoder_m],
-                            [self.vision_proj,self.vision_proj_m],
-                            [self.text_encoder,self.text_encoder_m],
-                            [self.text_proj,self.text_proj_m],
-                           ]       
-        self.copy_params()
-
-        # create the queue
-        self.register_buffer("image_queue", torch.randn(embed_dim, queue_size))
-        self.register_buffer("text_queue", torch.randn(embed_dim, queue_size))
-        self.register_buffer("queue_ptr", torch.zeros(1, dtype=torch.long))  
-
-        self.image_queue = nn.functional.normalize(self.image_queue, dim=0)
-        self.text_queue = nn.functional.normalize(self.text_queue, dim=0)
-        
-        self.queue_size = queue_size
-        self.momentum = momentum
-        self.temp = nn.Parameter(0.07*torch.ones([]))   
-        
-        # create the decoder
-        decoder_config = BertConfig.from_json_file(med_config)
-        decoder_config.encoder_width = vision_width        
-        self.text_decoder = BertLMHeadModel.from_pretrained('bert-base-uncased',config=decoder_config)    
-        self.text_decoder.resize_token_embeddings(len(self.tokenizer)) 
-        tie_encoder_decoder_weights(self.text_encoder,self.text_decoder.bert,'','/attention')
-        
-        
-    def forward(self, image, caption, alpha):
-        with torch.no_grad():
-            self.temp.clamp_(0.001,0.5)
-        
-        image_embeds = self.visual_encoder(image) 
-        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)        
-        image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1)          
-        
-        text = self.tokenizer(caption, padding='max_length', truncation=True, max_length=30, 
-                              return_tensors="pt").to(image.device)  
-        text_output = self.text_encoder(text.input_ids, attention_mask = text.attention_mask,                      
-                                        return_dict = True, mode = 'text')            
-        text_feat = F.normalize(self.text_proj(text_output.last_hidden_state[:,0,:]),dim=-1)                 
-             
-        # get momentum features
-        with torch.no_grad():
-            self._momentum_update()
-            image_embeds_m = self.visual_encoder_m(image) 
-            image_feat_m = F.normalize(self.vision_proj_m(image_embeds_m[:,0,:]),dim=-1)  
-            image_feat_all = torch.cat([image_feat_m.t(),self.image_queue.clone().detach()],dim=1)                   
-            
-            text_output_m = self.text_encoder_m(text.input_ids, attention_mask = text.attention_mask,                      
-                                                return_dict = True, mode = 'text')    
-            text_feat_m = F.normalize(self.text_proj_m(text_output_m.last_hidden_state[:,0,:]),dim=-1) 
-            text_feat_all = torch.cat([text_feat_m.t(),self.text_queue.clone().detach()],dim=1)
-
-            sim_i2t_m = image_feat_m @ text_feat_all / self.temp  
-            sim_t2i_m = text_feat_m @ image_feat_all / self.temp 
-
-            sim_targets = torch.zeros(sim_i2t_m.size()).to(image.device)
-            sim_targets.fill_diagonal_(1)          
-
-            sim_i2t_targets = alpha * F.softmax(sim_i2t_m, dim=1) + (1 - alpha) * sim_targets
-            sim_t2i_targets = alpha * F.softmax(sim_t2i_m, dim=1) + (1 - alpha) * sim_targets        
-
-        sim_i2t = image_feat @ text_feat_all / self.temp
-        sim_t2i = text_feat @ image_feat_all / self.temp
-                             
-        loss_i2t = -torch.sum(F.log_softmax(sim_i2t, dim=1)*sim_i2t_targets,dim=1).mean()
-        loss_t2i = -torch.sum(F.log_softmax(sim_t2i, dim=1)*sim_t2i_targets,dim=1).mean() 
-
-        loss_ita = (loss_i2t+loss_t2i)/2
-
-        self._dequeue_and_enqueue(image_feat_m, text_feat_m)        
-
-        ###============== Image-text Matching ===================###
-        encoder_input_ids = text.input_ids.clone()
-        encoder_input_ids[:,0] = self.tokenizer.enc_token_id
-        
-        # forward the positve image-text pair
-        bs = image.size(0)
-        output_pos = self.text_encoder(encoder_input_ids,
-                                       attention_mask = text.attention_mask,
-                                       encoder_hidden_states = image_embeds,
-                                       encoder_attention_mask = image_atts,      
-                                       return_dict = True,
-                                      )            
-        with torch.no_grad():       
-            weights_t2i = F.softmax(sim_t2i[:,:bs],dim=1)+1e-4 
-            weights_t2i.fill_diagonal_(0)            
-            weights_i2t = F.softmax(sim_i2t[:,:bs],dim=1)+1e-4  
-            weights_i2t.fill_diagonal_(0)   
-            
-        # select a negative image for each text
-        image_embeds_neg = []    
-        for b in range(bs):
-            neg_idx = torch.multinomial(weights_t2i[b], 1).item()
-            image_embeds_neg.append(image_embeds[neg_idx])
-        image_embeds_neg = torch.stack(image_embeds_neg,dim=0)   
-
-        # select a negative text for each image
-        text_ids_neg = []
-        text_atts_neg = []
-        for b in range(bs):
-            neg_idx = torch.multinomial(weights_i2t[b], 1).item()
-            text_ids_neg.append(encoder_input_ids[neg_idx])
-            text_atts_neg.append(text.attention_mask[neg_idx])
-
-        text_ids_neg = torch.stack(text_ids_neg,dim=0)   
-        text_atts_neg = torch.stack(text_atts_neg,dim=0)      
-
-        text_ids_all = torch.cat([encoder_input_ids, text_ids_neg],dim=0)     
-        text_atts_all = torch.cat([text.attention_mask, text_atts_neg],dim=0)     
-
-        image_embeds_all = torch.cat([image_embeds_neg,image_embeds],dim=0)
-        image_atts_all = torch.cat([image_atts,image_atts],dim=0)
-
-        output_neg = self.text_encoder(text_ids_all,
-                                       attention_mask = text_atts_all,
-                                       encoder_hidden_states = image_embeds_all,
-                                       encoder_attention_mask = image_atts_all,      
-                                       return_dict = True,
-                                      )                            
-
-        vl_embeddings = torch.cat([output_pos.last_hidden_state[:,0,:], output_neg.last_hidden_state[:,0,:]],dim=0)
-        vl_output = self.itm_head(vl_embeddings)            
-
-        itm_labels = torch.cat([torch.ones(bs,dtype=torch.long),torch.zeros(2*bs,dtype=torch.long)],
-                               dim=0).to(image.device)
-        loss_itm = F.cross_entropy(vl_output, itm_labels)  
-        
-        ##================= LM ========================##     
-        decoder_input_ids = text.input_ids.clone()      
-        decoder_input_ids[:,0] = self.tokenizer.bos_token_id
-        decoder_targets = decoder_input_ids.masked_fill(decoder_input_ids == self.tokenizer.pad_token_id, -100) 
-
-        decoder_output = self.text_decoder(decoder_input_ids, 
-                                           attention_mask = text.attention_mask, 
-                                           encoder_hidden_states = image_embeds,
-                                           encoder_attention_mask = image_atts,                  
-                                           labels = decoder_targets,
-                                           return_dict = True,   
-                                          )   
-          
-        loss_lm = decoder_output.loss                
-        return loss_ita, loss_itm, loss_lm
- 
-
-
-    @torch.no_grad()    
-    def copy_params(self):
-        for model_pair in self.model_pairs:           
-            for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()):
-                param_m.data.copy_(param.data)  # initialize
-                param_m.requires_grad = False  # not update by gradient    
-
-            
-    @torch.no_grad()        
-    def _momentum_update(self):
-        for model_pair in self.model_pairs:           
-            for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()):
-                param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum)
-
-                        
-    @torch.no_grad()
-    def _dequeue_and_enqueue(self, image_feat, text_feat):
-        # gather keys before updating queue
-        image_feats = concat_all_gather(image_feat)
-        text_feats = concat_all_gather(text_feat)
-
-        batch_size = image_feats.shape[0]
-
-        ptr = int(self.queue_ptr)
-        assert self.queue_size % batch_size == 0  # for simplicity
-
-        # replace the keys at ptr (dequeue and enqueue)
-        self.image_queue[:, ptr:ptr + batch_size] = image_feats.T
-        self.text_queue[:, ptr:ptr + batch_size] = text_feats.T
-        ptr = (ptr + batch_size) % self.queue_size  # move pointer
-
-        self.queue_ptr[0] = ptr 
-
-
-def blip_pretrain(**kwargs):
-    model = BLIP_Pretrain(**kwargs)
-    return model 
-
-
-@torch.no_grad()
-def concat_all_gather(tensor):
-    """
-    Performs all_gather operation on the provided tensors.
-    *** Warning ***: torch.distributed.all_gather has no gradient.
-    """
-    tensors_gather = [torch.ones_like(tensor)
-        for _ in range(torch.distributed.get_world_size())]
-    torch.distributed.all_gather(tensors_gather, tensor, async_op=False)
-
-    output = torch.cat(tensors_gather, dim=0)
-    return output     
-
-
-from typing import List
-def tie_encoder_decoder_weights(encoder: nn.Module, decoder: nn.Module, base_model_prefix: str, skip_key:str):
-    uninitialized_encoder_weights: List[str] = []
-    if decoder.__class__ != encoder.__class__:
-        print(
-            f"{decoder.__class__} and {encoder.__class__} are not equal. In this case make sure that all encoder weights are correctly initialized."
-        )
-
-    def tie_encoder_to_decoder_recursively(
-        decoder_pointer: nn.Module,
-        encoder_pointer: nn.Module,
-        module_name: str,
-        uninitialized_encoder_weights: List[str],
-        skip_key: str,
-        depth=0,
-    ):
-        assert isinstance(decoder_pointer, nn.Module) and isinstance(
-            encoder_pointer, nn.Module
-        ), f"{decoder_pointer} and {encoder_pointer} have to be of type torch.nn.Module"
-        if hasattr(decoder_pointer, "weight") and skip_key not in module_name:
-            assert hasattr(encoder_pointer, "weight")
-            encoder_pointer.weight = decoder_pointer.weight
-            if hasattr(decoder_pointer, "bias"):
-                assert hasattr(encoder_pointer, "bias")
-                encoder_pointer.bias = decoder_pointer.bias                
-            print(module_name+' is tied')    
-            return
-
-        encoder_modules = encoder_pointer._modules
-        decoder_modules = decoder_pointer._modules
-        if len(decoder_modules) > 0:
-            assert (
-                len(encoder_modules) > 0
-            ), f"Encoder module {encoder_pointer} does not match decoder module {decoder_pointer}"
-
-            all_encoder_weights = set([module_name + "/" + sub_name for sub_name in encoder_modules.keys()])
-            encoder_layer_pos = 0
-            for name, module in decoder_modules.items():
-                if name.isdigit():
-                    encoder_name = str(int(name) + encoder_layer_pos)
-                    decoder_name = name
-                    if not isinstance(decoder_modules[decoder_name], type(encoder_modules[encoder_name])) and len(
-                        encoder_modules
-                    ) != len(decoder_modules):
-                        # this can happen if the name corresponds to the position in a list module list of layers
-                        # in this case the decoder has added a cross-attention that the encoder does not have
-                        # thus skip this step and subtract one layer pos from encoder
-                        encoder_layer_pos -= 1
-                        continue
-                elif name not in encoder_modules:
-                    continue
-                elif depth > 500:
-                    raise ValueError(
-                        "Max depth of recursive function `tie_encoder_to_decoder` reached. It seems that there is a circular dependency between two or more `nn.Modules` of your model."
-                    )
-                else:
-                    decoder_name = encoder_name = name
-                tie_encoder_to_decoder_recursively(
-                    decoder_modules[decoder_name],
-                    encoder_modules[encoder_name],
-                    module_name + "/" + name,
-                    uninitialized_encoder_weights,
-                    skip_key,
-                    depth=depth + 1,
-                )
-                all_encoder_weights.remove(module_name + "/" + encoder_name)
-
-            uninitialized_encoder_weights += list(all_encoder_weights)
-
-    # tie weights recursively
-    tie_encoder_to_decoder_recursively(decoder, encoder, base_model_prefix, uninitialized_encoder_weights, skip_key)  

extras/BLIP/models/blip_retrieval.py

@@ -1,319 +0,0 @@
-from extras.BLIP.models.med import BertConfig, BertModel
-from transformers import BertTokenizer
-
-import torch
-from torch import nn
-import torch.nn.functional as F
-
-from extras.BLIP.models.blip import create_vit, init_tokenizer, load_checkpoint
-
-class BLIP_Retrieval(nn.Module):
-    def __init__(self,                 
-                 med_config = 'configs/med_config.json',  
-                 image_size = 384,
-                 vit = 'base',
-                 vit_grad_ckpt = False,
-                 vit_ckpt_layer = 0,                      
-                 embed_dim = 256,     
-                 queue_size = 57600,
-                 momentum = 0.995,
-                 negative_all_rank = False,
-                 ):
-        """
-        Args:
-            med_config (str): path for the mixture of encoder-decoder model's configuration file
-            image_size (int): input image size
-            vit (str): model size of vision transformer
-        """               
-        super().__init__()
-        
-        self.visual_encoder, vision_width = create_vit(vit,image_size, vit_grad_ckpt, vit_ckpt_layer)
-        self.tokenizer = init_tokenizer()   
-        med_config = BertConfig.from_json_file(med_config)
-        med_config.encoder_width = vision_width
-        self.text_encoder = BertModel(config=med_config, add_pooling_layer=False)          
-
-        text_width = self.text_encoder.config.hidden_size
-        
-        self.vision_proj = nn.Linear(vision_width, embed_dim)
-        self.text_proj = nn.Linear(text_width, embed_dim)
-
-        self.itm_head = nn.Linear(text_width, 2) 
-        
-        # create momentum encoders  
-        self.visual_encoder_m, vision_width = create_vit(vit,image_size)              
-        self.vision_proj_m = nn.Linear(vision_width, embed_dim)
-        self.text_encoder_m = BertModel(config=med_config, add_pooling_layer=False)    
-        self.text_proj_m = nn.Linear(text_width, embed_dim)
-        
-        self.model_pairs = [[self.visual_encoder,self.visual_encoder_m],
-                            [self.vision_proj,self.vision_proj_m],
-                            [self.text_encoder,self.text_encoder_m],
-                            [self.text_proj,self.text_proj_m],
-                           ]       
-        self.copy_params()
-
-        # create the queue
-        self.register_buffer("image_queue", torch.randn(embed_dim, queue_size))
-        self.register_buffer("text_queue", torch.randn(embed_dim, queue_size))
-        self.register_buffer("idx_queue", torch.full((1,queue_size),-100))
-        self.register_buffer("ptr_queue", torch.zeros(1, dtype=torch.long))  
-
-        self.image_queue = nn.functional.normalize(self.image_queue, dim=0)
-        self.text_queue = nn.functional.normalize(self.text_queue, dim=0)
-        
-        self.queue_size = queue_size
-        self.momentum = momentum
-        self.temp = nn.Parameter(0.07*torch.ones([]))   
-        
-        self.negative_all_rank = negative_all_rank
-        
-        
-    def forward(self, image, caption, alpha, idx):
-        with torch.no_grad():
-            self.temp.clamp_(0.001,0.5)
-        
-        image_embeds = self.visual_encoder(image) 
-        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)        
-        image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1)    
-        
-        text = self.tokenizer(caption, padding='max_length', truncation=True, max_length=35, 
-                              return_tensors="pt").to(image.device) 
-        
-        text_output = self.text_encoder(text.input_ids, attention_mask = text.attention_mask,                      
-                                        return_dict = True, mode = 'text')            
-        text_feat = F.normalize(self.text_proj(text_output.last_hidden_state[:,0,:]),dim=-1)        
-        
-        ###============== Image-text Contrastive Learning ===================###
-        idx = idx.view(-1,1)
-        idx_all = torch.cat([idx.t(), self.idx_queue.clone().detach()],dim=1)  
-        pos_idx = torch.eq(idx, idx_all).float()       
-        sim_targets = pos_idx / pos_idx.sum(1,keepdim=True)   
-        
-        # get momentum features
-        with torch.no_grad():
-            self._momentum_update()
-            image_embeds_m = self.visual_encoder_m(image) 
-            image_feat_m = F.normalize(self.vision_proj_m(image_embeds_m[:,0,:]),dim=-1)  
-            image_feat_m_all = torch.cat([image_feat_m.t(),self.image_queue.clone().detach()],dim=1)                   
-            
-            text_output_m = self.text_encoder_m(text.input_ids, attention_mask = text.attention_mask,                      
-                                                return_dict = True, mode = 'text')    
-            text_feat_m = F.normalize(self.text_proj_m(text_output_m.last_hidden_state[:,0,:]),dim=-1) 
-            text_feat_m_all = torch.cat([text_feat_m.t(),self.text_queue.clone().detach()],dim=1)
-
-            sim_i2t_m = image_feat_m @ text_feat_m_all / self.temp  
-            sim_t2i_m = text_feat_m @ image_feat_m_all / self.temp   
-
-            sim_i2t_targets = alpha * F.softmax(sim_i2t_m, dim=1) + (1 - alpha) * sim_targets
-            sim_t2i_targets = alpha * F.softmax(sim_t2i_m, dim=1) + (1 - alpha) * sim_targets        
-
-        sim_i2t = image_feat @ text_feat_m_all / self.temp 
-        sim_t2i = text_feat @ image_feat_m_all / self.temp 
-                             
-        loss_i2t = -torch.sum(F.log_softmax(sim_i2t, dim=1)*sim_i2t_targets,dim=1).mean()
-        loss_t2i = -torch.sum(F.log_softmax(sim_t2i, dim=1)*sim_t2i_targets,dim=1).mean() 
-
-        loss_ita = (loss_i2t+loss_t2i)/2
-        
-        idxs = concat_all_gather(idx)
-        self._dequeue_and_enqueue(image_feat_m, text_feat_m, idxs)        
-
-        ###============== Image-text Matching ===================###
-        encoder_input_ids = text.input_ids.clone()
-        encoder_input_ids[:,0] = self.tokenizer.enc_token_id
-
-        # forward the positve image-text pair
-        bs = image.size(0)
-        output_pos = self.text_encoder(encoder_input_ids,
-                                       attention_mask = text.attention_mask,
-                                       encoder_hidden_states = image_embeds,
-                                       encoder_attention_mask = image_atts,      
-                                       return_dict = True,
-                                      )  
-        
-        
-        if self.negative_all_rank:    
-            # compute sample similarity
-            with torch.no_grad():                
-                mask = torch.eq(idx, idxs.t())
-
-                image_feat_world = concat_all_gather(image_feat)
-                text_feat_world = concat_all_gather(text_feat)
-
-                sim_i2t = image_feat @ text_feat_world.t() / self.temp 
-                sim_t2i = text_feat @ image_feat_world.t() / self.temp 
-
-                weights_i2t = F.softmax(sim_i2t,dim=1)
-                weights_i2t.masked_fill_(mask, 0)            
-
-                weights_t2i = F.softmax(sim_t2i,dim=1)
-                weights_t2i.masked_fill_(mask, 0)     
-
-            image_embeds_world = all_gather_with_grad(image_embeds) 
-
-            # select a negative image (from all ranks) for each text
-            image_embeds_neg = []    
-            for b in range(bs):
-                neg_idx = torch.multinomial(weights_t2i[b], 1).item()
-                image_embeds_neg.append(image_embeds_world[neg_idx])
-            image_embeds_neg = torch.stack(image_embeds_neg,dim=0)   
-
-            # select a negative text (from all ranks) for each image
-            input_ids_world = concat_all_gather(encoder_input_ids)
-            att_mask_world = concat_all_gather(text.attention_mask)        
-
-            text_ids_neg = []
-            text_atts_neg = []
-            for b in range(bs):
-                neg_idx = torch.multinomial(weights_i2t[b], 1).item()
-                text_ids_neg.append(input_ids_world[neg_idx])
-                text_atts_neg.append(att_mask_world[neg_idx])
-                
-        else:
-            with torch.no_grad():                
-                mask = torch.eq(idx, idx.t())
-                
-                sim_i2t = image_feat @ text_feat.t() / self.temp 
-                sim_t2i = text_feat @ image_feat.t() / self.temp 
-
-                weights_i2t = F.softmax(sim_i2t,dim=1)
-                weights_i2t.masked_fill_(mask, 0)            
-
-                weights_t2i = F.softmax(sim_t2i,dim=1)
-                weights_t2i.masked_fill_(mask, 0)     
-
-            # select a negative image (from same rank) for each text
-            image_embeds_neg = []    
-            for b in range(bs):
-                neg_idx = torch.multinomial(weights_t2i[b], 1).item()
-                image_embeds_neg.append(image_embeds[neg_idx])
-            image_embeds_neg = torch.stack(image_embeds_neg,dim=0)   
-
-            # select a negative text (from same rank) for each image    
-            text_ids_neg = []
-            text_atts_neg = []
-            for b in range(bs):
-                neg_idx = torch.multinomial(weights_i2t[b], 1).item()
-                text_ids_neg.append(encoder_input_ids[neg_idx])
-                text_atts_neg.append(text.attention_mask[neg_idx])            
-            
-        text_ids_neg = torch.stack(text_ids_neg,dim=0)   
-        text_atts_neg = torch.stack(text_atts_neg,dim=0)      
-
-        text_ids_all = torch.cat([encoder_input_ids, text_ids_neg],dim=0)     
-        text_atts_all = torch.cat([text.attention_mask, text_atts_neg],dim=0)     
-
-        image_embeds_all = torch.cat([image_embeds_neg,image_embeds],dim=0)
-        image_atts_all = torch.cat([image_atts,image_atts],dim=0)
-
-        output_neg = self.text_encoder(text_ids_all,
-                                       attention_mask = text_atts_all,
-                                       encoder_hidden_states = image_embeds_all,
-                                       encoder_attention_mask = image_atts_all,      
-                                       return_dict = True,
-                                      )                         
-          
-
-        vl_embeddings = torch.cat([output_pos.last_hidden_state[:,0,:], output_neg.last_hidden_state[:,0,:]],dim=0)
-        vl_output = self.itm_head(vl_embeddings)            
-
-        itm_labels = torch.cat([torch.ones(bs,dtype=torch.long),torch.zeros(2*bs,dtype=torch.long)],
-                               dim=0).to(image.device)
-        loss_itm = F.cross_entropy(vl_output, itm_labels)     
-
-        return loss_ita, loss_itm 
- 
-
-    @torch.no_grad()    
-    def copy_params(self):
-        for model_pair in self.model_pairs:           
-            for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()):
-                param_m.data.copy_(param.data)  # initialize
-                param_m.requires_grad = False  # not update by gradient    
-
-            
-    @torch.no_grad()        
-    def _momentum_update(self):
-        for model_pair in self.model_pairs:           
-            for param, param_m in zip(model_pair[0].parameters(), model_pair[1].parameters()):
-                param_m.data = param_m.data * self.momentum + param.data * (1. - self.momentum)
-                
-                
-    @torch.no_grad()
-    def _dequeue_and_enqueue(self, image_feat, text_feat, idxs):
-        # gather keys before updating queue
-        image_feats = concat_all_gather(image_feat)
-        text_feats = concat_all_gather(text_feat)
-        
-
-        batch_size = image_feats.shape[0]
-
-        ptr = int(self.ptr_queue)
-        assert self.queue_size % batch_size == 0  # for simplicity
-
-        # replace the keys at ptr (dequeue and enqueue)
-        self.image_queue[:, ptr:ptr + batch_size] = image_feats.T
-        self.text_queue[:, ptr:ptr + batch_size] = text_feats.T
-        self.idx_queue[:, ptr:ptr + batch_size] = idxs.T
-        ptr = (ptr + batch_size) % self.queue_size # move pointer
-
-        self.ptr_queue[0] = ptr  
-
-
-def blip_retrieval(pretrained='',**kwargs):
-    model = BLIP_Retrieval(**kwargs)
-    if pretrained:
-        model,msg = load_checkpoint(model,pretrained)
-        print("missing keys:")
-        print(msg.missing_keys)
-    return model 
-
-
-@torch.no_grad()
-def concat_all_gather(tensor):
-    """
-    Performs all_gather operation on the provided tensors.
-    *** Warning ***: torch.distributed.all_gather has no gradient.
-    """
-    tensors_gather = [torch.ones_like(tensor)
-        for _ in range(torch.distributed.get_world_size())]
-    torch.distributed.all_gather(tensors_gather, tensor, async_op=False)
-
-    output = torch.cat(tensors_gather, dim=0)
-    return output      
-
-
-class GatherLayer(torch.autograd.Function):
-    """
-    Gather tensors from all workers with support for backward propagation:
-    This implementation does not cut the gradients as torch.distributed.all_gather does.
-    """
-
-    @staticmethod
-    def forward(ctx, x):
-        output = [torch.zeros_like(x) for _ in range(torch.distributed.get_world_size())]
-        torch.distributed.all_gather(output, x)
-        return tuple(output)
-
-    @staticmethod
-    def backward(ctx, *grads):
-        all_gradients = torch.stack(grads)
-        torch.distributed.all_reduce(all_gradients)
-        return all_gradients[torch.distributed.get_rank()]
-
-
-def all_gather_with_grad(tensors):
-    """
-    Performs all_gather operation on the provided tensors.
-    Graph remains connected for backward grad computation.
-    """
-    # Queue the gathered tensors
-    world_size = torch.distributed.get_world_size()
-    # There is no need for reduction in the single-proc case
-    if world_size == 1:
-        return tensors
-
-    tensor_all = GatherLayer.apply(tensors)
-
-    return torch.cat(tensor_all, dim=0)

extras/BLIP/models/blip_vqa.py

@@ -1,186 +0,0 @@
-from extras.BLIP.models.med import BertConfig, BertModel, BertLMHeadModel
-from extras.BLIP.models.blip import create_vit, init_tokenizer, load_checkpoint
-
-import torch
-from torch import nn
-import torch.nn.functional as F
-from transformers import BertTokenizer
-import numpy as np
-
-class BLIP_VQA(nn.Module):
-    def __init__(self,                 
-                 med_config = 'configs/med_config.json',  
-                 image_size = 480,
-                 vit = 'base',
-                 vit_grad_ckpt = False,
-                 vit_ckpt_layer = 0,                   
-                 ):
-        """
-        Args:
-            med_config (str): path for the mixture of encoder-decoder model's configuration file
-            image_size (int): input image size
-            vit (str): model size of vision transformer
-        """               
-        super().__init__()
-        
-        self.visual_encoder, vision_width = create_vit(vit, image_size, vit_grad_ckpt, vit_ckpt_layer, drop_path_rate=0.1)
-        self.tokenizer = init_tokenizer()  
-        
-        encoder_config = BertConfig.from_json_file(med_config)
-        encoder_config.encoder_width = vision_width
-        self.text_encoder = BertModel(config=encoder_config, add_pooling_layer=False) 
-        
-        decoder_config = BertConfig.from_json_file(med_config)        
-        self.text_decoder = BertLMHeadModel(config=decoder_config)          
-
-
-    def forward(self, image, question, answer=None, n=None, weights=None, train=True, inference='rank', k_test=128):
-        
-        image_embeds = self.visual_encoder(image) 
-        image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image.device)
-        
-        question = self.tokenizer(question, padding='longest', truncation=True, max_length=35, 
-                                  return_tensors="pt").to(image.device) 
-        question.input_ids[:,0] = self.tokenizer.enc_token_id
-        
-        if train:               
-            '''
-            n: number of answers for each question
-            weights: weight for each answer
-            '''                     
-            answer = self.tokenizer(answer, padding='longest', return_tensors="pt").to(image.device) 
-            answer.input_ids[:,0] = self.tokenizer.bos_token_id
-            answer_targets = answer.input_ids.masked_fill(answer.input_ids == self.tokenizer.pad_token_id, -100)      
-
-            question_output = self.text_encoder(question.input_ids, 
-                                                attention_mask = question.attention_mask, 
-                                                encoder_hidden_states = image_embeds,
-                                                encoder_attention_mask = image_atts,                             
-                                                return_dict = True)    
-
-            question_states = []                
-            question_atts = []  
-            for b, n in enumerate(n):
-                question_states += [question_output.last_hidden_state[b]]*n
-                question_atts += [question.attention_mask[b]]*n                
-            question_states = torch.stack(question_states,0)    
-            question_atts = torch.stack(question_atts,0)     
-
-            answer_output = self.text_decoder(answer.input_ids, 
-                                              attention_mask = answer.attention_mask, 
-                                              encoder_hidden_states = question_states,
-                                              encoder_attention_mask = question_atts,                  
-                                              labels = answer_targets,
-                                              return_dict = True,   
-                                              reduction = 'none',
-                                             )      
-            
-            loss = weights * answer_output.loss
-            loss = loss.sum()/image.size(0)
-
-            return loss
-            
-
-        else: 
-            question_output = self.text_encoder(question.input_ids, 
-                                                attention_mask = question.attention_mask, 
-                                                encoder_hidden_states = image_embeds,
-                                                encoder_attention_mask = image_atts,                                    
-                                                return_dict = True) 
-            
-            if inference=='generate':
-                num_beams = 3
-                question_states = question_output.last_hidden_state.repeat_interleave(num_beams,dim=0)
-                question_atts = torch.ones(question_states.size()[:-1],dtype=torch.long).to(question_states.device)
-                model_kwargs = {"encoder_hidden_states": question_states, "encoder_attention_mask":question_atts}
-                
-                bos_ids = torch.full((image.size(0),1),fill_value=self.tokenizer.bos_token_id,device=image.device)
-                
-                outputs = self.text_decoder.generate(input_ids=bos_ids,
-                                                     max_length=10,
-                                                     min_length=1,
-                                                     num_beams=num_beams,
-                                                     eos_token_id=self.tokenizer.sep_token_id,
-                                                     pad_token_id=self.tokenizer.pad_token_id, 
-                                                     **model_kwargs)
-                
-                answers = []    
-                for output in outputs:
-                    answer = self.tokenizer.decode(output, skip_special_tokens=True)    
-                    answers.append(answer)
-                return answers
-            
-            elif inference=='rank':
-                max_ids = self.rank_answer(question_output.last_hidden_state, question.attention_mask, 
-                                           answer.input_ids, answer.attention_mask, k_test) 
-                return max_ids
- 
-                
-                
-    def rank_answer(self, question_states, question_atts, answer_ids, answer_atts, k):
-        
-        num_ques = question_states.size(0)
-        start_ids = answer_ids[0,0].repeat(num_ques,1) # bos token
-        
-        start_output = self.text_decoder(start_ids, 
-                                         encoder_hidden_states = question_states,
-                                         encoder_attention_mask = question_atts,                                      
-                                         return_dict = True,
-                                         reduction = 'none')              
-        logits = start_output.logits[:,0,:] # first token's logit
-        
-        # topk_probs: top-k probability 
-        # topk_ids: [num_question, k]        
-        answer_first_token = answer_ids[:,1]
-        prob_first_token = F.softmax(logits,dim=1).index_select(dim=1, index=answer_first_token) 
-        topk_probs, topk_ids = prob_first_token.topk(k,dim=1) 
-        
-        # answer input: [num_question*k, answer_len]                 
-        input_ids = []
-        input_atts = []
-        for b, topk_id in enumerate(topk_ids):
-            input_ids.append(answer_ids.index_select(dim=0, index=topk_id))
-            input_atts.append(answer_atts.index_select(dim=0, index=topk_id))
-        input_ids = torch.cat(input_ids,dim=0)  
-        input_atts = torch.cat(input_atts,dim=0)  
-
-        targets_ids = input_ids.masked_fill(input_ids == self.tokenizer.pad_token_id, -100)
-
-        # repeat encoder's output for top-k answers
-        question_states = tile(question_states, 0, k)
-        question_atts = tile(question_atts, 0, k)
-        
-        output = self.text_decoder(input_ids, 
-                                   attention_mask = input_atts, 
-                                   encoder_hidden_states = question_states,
-                                   encoder_attention_mask = question_atts,     
-                                   labels = targets_ids,
-                                   return_dict = True, 
-                                   reduction = 'none')   
-        
-        log_probs_sum = -output.loss
-        log_probs_sum = log_probs_sum.view(num_ques,k)
-
-        max_topk_ids = log_probs_sum.argmax(dim=1) 
-        max_ids = topk_ids[max_topk_ids>=0,max_topk_ids]
-
-        return max_ids
-    
-    
-def blip_vqa(pretrained='',**kwargs):
-    model = BLIP_VQA(**kwargs)
-    if pretrained:
-        model,msg = load_checkpoint(model,pretrained)
-#         assert(len(msg.missing_keys)==0)
-    return model  
-
-
-def tile(x, dim, n_tile):
-    init_dim = x.size(dim)
-    repeat_idx = [1] * x.dim()
-    repeat_idx[dim] = n_tile
-    x = x.repeat(*(repeat_idx))
-    order_index = torch.LongTensor(np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)]))
-    return torch.index_select(x, dim, order_index.to(x.device))    
-        
-        

extras/BLIP/models/med.py

@@ -1,955 +0,0 @@
-'''
- * Copyright (c) 2022, salesforce.com, inc.
- * All rights reserved.
- * SPDX-License-Identifier: BSD-3-Clause
- * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
- * By Junnan Li
- * Based on huggingface code base
- * https://github.com/huggingface/transformers/blob/v4.15.0/src/transformers/models/bert
-'''
-
-import math
-import os
-import warnings
-from dataclasses import dataclass
-from typing import Optional, Tuple
-
-import torch
-from torch import Tensor, device, dtype, nn
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import CrossEntropyLoss
-import torch.nn.functional as F
-
-from transformers.activations import ACT2FN
-from transformers.file_utils import (
-    ModelOutput,
-)
-from transformers.modeling_outputs import (
-    BaseModelOutputWithPastAndCrossAttentions,
-    BaseModelOutputWithPoolingAndCrossAttentions,
-    CausalLMOutputWithCrossAttentions,
-    MaskedLMOutput,
-    MultipleChoiceModelOutput,
-    NextSentencePredictorOutput,
-    QuestionAnsweringModelOutput,
-    SequenceClassifierOutput,
-    TokenClassifierOutput,
-)
-from transformers.modeling_utils import (
-    PreTrainedModel,
-    apply_chunking_to_forward,
-    find_pruneable_heads_and_indices,
-    prune_linear_layer,
-)
-from transformers.utils import logging
-from transformers.models.bert.configuration_bert import BertConfig
-
-
-logger = logging.get_logger(__name__)
-
-
-class BertEmbeddings(nn.Module):
-    """Construct the embeddings from word and position embeddings."""
-
-    def __init__(self, config):
-        super().__init__()
-        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
-        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
-
-        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
-        # any TensorFlow checkpoint file
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)
-
-        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
-        self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
-        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
-        
-        self.config = config
-
-    def forward(
-        self, input_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0
-    ):
-        if input_ids is not None:
-            input_shape = input_ids.size()
-        else:
-            input_shape = inputs_embeds.size()[:-1]
-
-        seq_length = input_shape[1]
-
-        if position_ids is None:
-            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
-
-        if inputs_embeds is None:
-            inputs_embeds = self.word_embeddings(input_ids)
-
-        embeddings = inputs_embeds
-
-        if self.position_embedding_type == "absolute":
-            position_embeddings = self.position_embeddings(position_ids)
-            embeddings += position_embeddings
-        embeddings = self.LayerNorm(embeddings)
-        embeddings = self.dropout(embeddings)
-        return embeddings
-
-
-class BertSelfAttention(nn.Module):
-    def __init__(self, config, is_cross_attention):
-        super().__init__()
-        self.config = config
-        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
-            raise ValueError(
-                "The hidden size (%d) is not a multiple of the number of attention "
-                "heads (%d)" % (config.hidden_size, config.num_attention_heads)
-            )
-        
-        self.num_attention_heads = config.num_attention_heads
-        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
-        self.all_head_size = self.num_attention_heads * self.attention_head_size
-
-        self.query = nn.Linear(config.hidden_size, self.all_head_size)
-        if is_cross_attention:
-            self.key = nn.Linear(config.encoder_width, self.all_head_size)
-            self.value = nn.Linear(config.encoder_width, self.all_head_size)
-        else:
-            self.key = nn.Linear(config.hidden_size, self.all_head_size)
-            self.value = nn.Linear(config.hidden_size, self.all_head_size)
-
-        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
-        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
-        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
-            self.max_position_embeddings = config.max_position_embeddings
-            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)
-        self.save_attention = False   
-            
-    def save_attn_gradients(self, attn_gradients):
-        self.attn_gradients = attn_gradients
-        
-    def get_attn_gradients(self):
-        return self.attn_gradients
-    
-    def save_attention_map(self, attention_map):
-        self.attention_map = attention_map
-        
-    def get_attention_map(self):
-        return self.attention_map
-    
-    def transpose_for_scores(self, x):
-        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
-        x = x.view(*new_x_shape)
-        return x.permute(0, 2, 1, 3)
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_value=None,
-        output_attentions=False,
-    ):
-        mixed_query_layer = self.query(hidden_states)
-
-        # If this is instantiated as a cross-attention module, the keys
-        # and values come from an encoder; the attention mask needs to be
-        # such that the encoder's padding tokens are not attended to.
-        is_cross_attention = encoder_hidden_states is not None
-
-        if is_cross_attention:
-            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
-            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
-            attention_mask = encoder_attention_mask
-        elif past_key_value is not None:
-            key_layer = self.transpose_for_scores(self.key(hidden_states))
-            value_layer = self.transpose_for_scores(self.value(hidden_states))
-            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
-            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
-        else:
-            key_layer = self.transpose_for_scores(self.key(hidden_states))
-            value_layer = self.transpose_for_scores(self.value(hidden_states))
-
-        query_layer = self.transpose_for_scores(mixed_query_layer)
-
-        past_key_value = (key_layer, value_layer)
-
-        # Take the dot product between "query" and "key" to get the raw attention scores.
-        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
-
-        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
-            seq_length = hidden_states.size()[1]
-            position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
-            position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
-            distance = position_ids_l - position_ids_r
-            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
-            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility
-
-            if self.position_embedding_type == "relative_key":
-                relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
-                attention_scores = attention_scores + relative_position_scores
-            elif self.position_embedding_type == "relative_key_query":
-                relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
-                relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
-                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
-
-        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
-        if attention_mask is not None:
-            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
-            attention_scores = attention_scores + attention_mask
-
-        # Normalize the attention scores to probabilities.
-        attention_probs = nn.Softmax(dim=-1)(attention_scores)
-        
-        if is_cross_attention and self.save_attention:
-            self.save_attention_map(attention_probs)
-            attention_probs.register_hook(self.save_attn_gradients)         
-
-        # This is actually dropping out entire tokens to attend to, which might
-        # seem a bit unusual, but is taken from the original Transformer paper.
-        attention_probs_dropped = self.dropout(attention_probs)
-
-        # Mask heads if we want to
-        if head_mask is not None:
-            attention_probs_dropped = attention_probs_dropped * head_mask
-
-        context_layer = torch.matmul(attention_probs_dropped, value_layer)
-
-        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
-        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
-        context_layer = context_layer.view(*new_context_layer_shape)
-
-        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
-
-        outputs = outputs + (past_key_value,)
-        return outputs
-
-
-class BertSelfOutput(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)
-
-    def forward(self, hidden_states, input_tensor):
-        hidden_states = self.dense(hidden_states)
-        hidden_states = self.dropout(hidden_states)
-        hidden_states = self.LayerNorm(hidden_states + input_tensor)
-        return hidden_states
-
-
-class BertAttention(nn.Module):
-    def __init__(self, config, is_cross_attention=False):
-        super().__init__()
-        self.self = BertSelfAttention(config, is_cross_attention)
-        self.output = BertSelfOutput(config)
-        self.pruned_heads = set()
-
-    def prune_heads(self, heads):
-        if len(heads) == 0:
-            return
-        heads, index = find_pruneable_heads_and_indices(
-            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
-        )
-
-        # Prune linear layers
-        self.self.query = prune_linear_layer(self.self.query, index)
-        self.self.key = prune_linear_layer(self.self.key, index)
-        self.self.value = prune_linear_layer(self.self.value, index)
-        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
-
-        # Update hyper params and store pruned heads
-        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
-        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
-        self.pruned_heads = self.pruned_heads.union(heads)
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_value=None,
-        output_attentions=False,
-    ):
-        self_outputs = self.self(
-            hidden_states,
-            attention_mask,
-            head_mask,
-            encoder_hidden_states,
-            encoder_attention_mask,
-            past_key_value,
-            output_attentions,
-        )
-        attention_output = self.output(self_outputs[0], hidden_states)
-        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
-        return outputs
-
-
-class BertIntermediate(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
-        if isinstance(config.hidden_act, str):
-            self.intermediate_act_fn = ACT2FN[config.hidden_act]
-        else:
-            self.intermediate_act_fn = config.hidden_act
-
-    def forward(self, hidden_states):
-        hidden_states = self.dense(hidden_states)
-        hidden_states = self.intermediate_act_fn(hidden_states)
-        return hidden_states
-
-
-class BertOutput(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)
-
-    def forward(self, hidden_states, input_tensor):
-        hidden_states = self.dense(hidden_states)
-        hidden_states = self.dropout(hidden_states)
-        hidden_states = self.LayerNorm(hidden_states + input_tensor)
-        return hidden_states
-
-
-class BertLayer(nn.Module):
-    def __init__(self, config, layer_num):
-        super().__init__()
-        self.config = config
-        self.chunk_size_feed_forward = config.chunk_size_feed_forward
-        self.seq_len_dim = 1
-        self.attention = BertAttention(config)      
-        self.layer_num = layer_num          
-        if self.config.add_cross_attention:
-            self.crossattention = BertAttention(config, is_cross_attention=self.config.add_cross_attention)
-        self.intermediate = BertIntermediate(config)
-        self.output = BertOutput(config)
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_value=None,
-        output_attentions=False,
-        mode=None,
-    ):
-        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
-        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
-        self_attention_outputs = self.attention(
-            hidden_states,
-            attention_mask,
-            head_mask,
-            output_attentions=output_attentions,
-            past_key_value=self_attn_past_key_value,
-        )
-        attention_output = self_attention_outputs[0]
-
-        outputs = self_attention_outputs[1:-1]
-        present_key_value = self_attention_outputs[-1]
-
-        if mode=='multimodal':
-            assert encoder_hidden_states is not None, "encoder_hidden_states must be given for cross-attention layers"
-
-            cross_attention_outputs = self.crossattention(
-                attention_output,
-                attention_mask,
-                head_mask,
-                encoder_hidden_states,
-                encoder_attention_mask,
-                output_attentions=output_attentions,
-            )
-            attention_output = cross_attention_outputs[0]
-            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights                               
-        layer_output = apply_chunking_to_forward(
-            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
-        )
-        outputs = (layer_output,) + outputs
-
-        outputs = outputs + (present_key_value,)
-
-        return outputs
-
-    def feed_forward_chunk(self, attention_output):
-        intermediate_output = self.intermediate(attention_output)
-        layer_output = self.output(intermediate_output, attention_output)
-        return layer_output
-
-
-class BertEncoder(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.config = config
-        self.layer = nn.ModuleList([BertLayer(config,i) for i in range(config.num_hidden_layers)])
-        self.gradient_checkpointing = False
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_values=None,
-        use_cache=None,
-        output_attentions=False,
-        output_hidden_states=False,
-        return_dict=True,
-        mode='multimodal',
-    ):
-        all_hidden_states = () if output_hidden_states else None
-        all_self_attentions = () if output_attentions else None
-        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
-
-        next_decoder_cache = () if use_cache else None
-               
-        for i in range(self.config.num_hidden_layers):
-            layer_module = self.layer[i]
-            if output_hidden_states:
-                all_hidden_states = all_hidden_states + (hidden_states,)
-
-            layer_head_mask = head_mask[i] if head_mask is not None else None
-            past_key_value = past_key_values[i] if past_key_values is not None else None
-
-            if self.gradient_checkpointing and self.training:
-
-                if use_cache:
-                    logger.warn(
-                        "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
-                    )
-                    use_cache = False
-
-                def create_custom_forward(module):
-                    def custom_forward(*inputs):
-                        return module(*inputs, past_key_value, output_attentions)
-
-                    return custom_forward
-
-                layer_outputs = torch.utils.checkpoint.checkpoint(
-                    create_custom_forward(layer_module),
-                    hidden_states,
-                    attention_mask,
-                    layer_head_mask,
-                    encoder_hidden_states,
-                    encoder_attention_mask,
-                    mode=mode,
-                )
-            else:
-                layer_outputs = layer_module(
-                    hidden_states,
-                    attention_mask,
-                    layer_head_mask,
-                    encoder_hidden_states,
-                    encoder_attention_mask,
-                    past_key_value,
-                    output_attentions,
-                    mode=mode,
-                )
-
-            hidden_states = layer_outputs[0]
-            if use_cache:
-                next_decoder_cache += (layer_outputs[-1],)
-            if output_attentions:
-                all_self_attentions = all_self_attentions + (layer_outputs[1],)
-
-        if output_hidden_states:
-            all_hidden_states = all_hidden_states + (hidden_states,)
-
-        if not return_dict:
-            return tuple(
-                v
-                for v in [
-                    hidden_states,
-                    next_decoder_cache,
-                    all_hidden_states,
-                    all_self_attentions,
-                    all_cross_attentions,
-                ]
-                if v is not None
-            )
-        return BaseModelOutputWithPastAndCrossAttentions(
-            last_hidden_state=hidden_states,
-            past_key_values=next_decoder_cache,
-            hidden_states=all_hidden_states,
-            attentions=all_self_attentions,
-            cross_attentions=all_cross_attentions,
-        )
-
-
-class BertPooler(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        self.activation = nn.Tanh()
-
-    def forward(self, hidden_states):
-        # We "pool" the model by simply taking the hidden state corresponding
-        # to the first token.
-        first_token_tensor = hidden_states[:, 0]
-        pooled_output = self.dense(first_token_tensor)
-        pooled_output = self.activation(pooled_output)
-        return pooled_output
-
-
-class BertPredictionHeadTransform(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        if isinstance(config.hidden_act, str):
-            self.transform_act_fn = ACT2FN[config.hidden_act]
-        else:
-            self.transform_act_fn = config.hidden_act
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-
-    def forward(self, hidden_states):
-        hidden_states = self.dense(hidden_states)
-        hidden_states = self.transform_act_fn(hidden_states)
-        hidden_states = self.LayerNorm(hidden_states)
-        return hidden_states
-
-
-class BertLMPredictionHead(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.transform = BertPredictionHeadTransform(config)
-
-        # The output weights are the same as the input embeddings, but there is
-        # an output-only bias for each token.
-        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
-
-        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
-
-        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
-        self.decoder.bias = self.bias
-
-    def forward(self, hidden_states):
-        hidden_states = self.transform(hidden_states)
-        hidden_states = self.decoder(hidden_states)
-        return hidden_states
-
-
-class BertOnlyMLMHead(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.predictions = BertLMPredictionHead(config)
-
-    def forward(self, sequence_output):
-        prediction_scores = self.predictions(sequence_output)
-        return prediction_scores
-
-
-class BertPreTrainedModel(PreTrainedModel):
-    """
-    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
-    models.
-    """
-
-    config_class = BertConfig
-    base_model_prefix = "bert"
-    _keys_to_ignore_on_load_missing = [r"position_ids"]
-
-    def _init_weights(self, module):
-        """ Initialize the weights """
-        if isinstance(module, (nn.Linear, nn.Embedding)):
-            # Slightly different from the TF version which uses truncated_normal for initialization
-            # cf https://github.com/pytorch/pytorch/pull/5617
-            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
-        elif isinstance(module, nn.LayerNorm):
-            module.bias.data.zero_()
-            module.weight.data.fill_(1.0)
-        if isinstance(module, nn.Linear) and module.bias is not None:
-            module.bias.data.zero_()
-
-
-class BertModel(BertPreTrainedModel):
-    """
-    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
-    cross-attention is added between the self-attention layers, following the architecture described in `Attention is
-    all you need <https://arxiv.org/abs/1706.03762>`__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
-    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
-    argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an
-    input to the forward pass.
-    """
-
-    def __init__(self, config, add_pooling_layer=True):
-        super().__init__(config)
-        self.config = config
-
-        self.embeddings = BertEmbeddings(config)
-        
-        self.encoder = BertEncoder(config)
-
-        self.pooler = BertPooler(config) if add_pooling_layer else None
-
-        self.init_weights()
- 
-
-    def get_input_embeddings(self):
-        return self.embeddings.word_embeddings
-
-    def set_input_embeddings(self, value):
-        self.embeddings.word_embeddings = value
-
-    def _prune_heads(self, heads_to_prune):
-        """
-        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
-        class PreTrainedModel
-        """
-        for layer, heads in heads_to_prune.items():
-            self.encoder.layer[layer].attention.prune_heads(heads)
-
-    
-    def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool) -> Tensor:
-        """
-        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
-
-        Arguments:
-            attention_mask (:obj:`torch.Tensor`):
-                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
-            input_shape (:obj:`Tuple[int]`):
-                The shape of the input to the model.
-            device: (:obj:`torch.device`):
-                The device of the input to the model.
-
-        Returns:
-            :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`.
-        """
-        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
-        # ourselves in which case we just need to make it broadcastable to all heads.
-        if attention_mask.dim() == 3:
-            extended_attention_mask = attention_mask[:, None, :, :]
-        elif attention_mask.dim() == 2:
-            # Provided a padding mask of dimensions [batch_size, seq_length]
-            # - if the model is a decoder, apply a causal mask in addition to the padding mask
-            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
-            if is_decoder:
-                batch_size, seq_length = input_shape
-
-                seq_ids = torch.arange(seq_length, device=device)
-                causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
-                # in case past_key_values are used we need to add a prefix ones mask to the causal mask
-                # causal and attention masks must have same type with pytorch version < 1.3
-                causal_mask = causal_mask.to(attention_mask.dtype)
-   
-                if causal_mask.shape[1] < attention_mask.shape[1]:
-                    prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
-                    causal_mask = torch.cat(
-                        [
-                            torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype),
-                            causal_mask,
-                        ],
-                        axis=-1,
-                    )                     
-
-                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
-            else:
-                extended_attention_mask = attention_mask[:, None, None, :]
-        else:
-            raise ValueError(
-                "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(
-                    input_shape, attention_mask.shape
-                )
-            )
-
-        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
-        # masked positions, this operation will create a tensor which is 0.0 for
-        # positions we want to attend and -10000.0 for masked positions.
-        # Since we are adding it to the raw scores before the softmax, this is
-        # effectively the same as removing these entirely.
-        extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
-        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
-        return extended_attention_mask
-    
-    def forward(
-        self,
-        input_ids=None,
-        attention_mask=None,
-        position_ids=None,
-        head_mask=None,
-        inputs_embeds=None,
-        encoder_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_values=None,
-        use_cache=None,
-        output_attentions=None,
-        output_hidden_states=None,
-        return_dict=None,
-        is_decoder=False,
-        mode='multimodal',
-    ):
-        r"""
-        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
-            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
-            the model is configured as a decoder.
-        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
-            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
-            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
-            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
-            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
-            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
-            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
-        use_cache (:obj:`bool`, `optional`):
-            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
-            decoding (see :obj:`past_key_values`).
-        """
-        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        if is_decoder:
-            use_cache = use_cache if use_cache is not None else self.config.use_cache
-        else:
-            use_cache = False
-
-        if input_ids is not None and inputs_embeds is not None:
-            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
-        elif input_ids is not None:
-            input_shape = input_ids.size()
-            batch_size, seq_length = input_shape
-            device = input_ids.device
-        elif inputs_embeds is not None:
-            input_shape = inputs_embeds.size()[:-1]
-            batch_size, seq_length = input_shape
-            device = inputs_embeds.device
-        elif encoder_embeds is not None:    
-            input_shape = encoder_embeds.size()[:-1]
-            batch_size, seq_length = input_shape 
-            device = encoder_embeds.device
-        else:
-            raise ValueError("You have to specify either input_ids or inputs_embeds or encoder_embeds")
-
-        # past_key_values_length
-        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0
-
-        if attention_mask is None:
-            attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device)
-            
-        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
-        # ourselves in which case we just need to make it broadcastable to all heads.
-        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, 
-                                                                                 device, is_decoder)
-
-        # If a 2D or 3D attention mask is provided for the cross-attention
-        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
-        if encoder_hidden_states is not None:
-            if type(encoder_hidden_states) == list:
-                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size()
-            else:
-                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
-            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
-            
-            if type(encoder_attention_mask) == list:
-                encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask]
-            elif encoder_attention_mask is None:
-                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
-                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
-            else:    
-                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
-        else:
-            encoder_extended_attention_mask = None
-
-        # Prepare head mask if needed
-        # 1.0 in head_mask indicate we keep the head
-        # attention_probs has shape bsz x n_heads x N x N
-        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
-        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
-        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
-        
-        if encoder_embeds is None:
-            embedding_output = self.embeddings(
-                input_ids=input_ids,
-                position_ids=position_ids,
-                inputs_embeds=inputs_embeds,
-                past_key_values_length=past_key_values_length,
-            )
-        else:
-            embedding_output = encoder_embeds
-            
-        encoder_outputs = self.encoder(
-            embedding_output,
-            attention_mask=extended_attention_mask,
-            head_mask=head_mask,
-            encoder_hidden_states=encoder_hidden_states,
-            encoder_attention_mask=encoder_extended_attention_mask,
-            past_key_values=past_key_values,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-            mode=mode,
-        )
-        sequence_output = encoder_outputs[0]
-        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
-
-        if not return_dict:
-            return (sequence_output, pooled_output) + encoder_outputs[1:]
-
-        return BaseModelOutputWithPoolingAndCrossAttentions(
-            last_hidden_state=sequence_output,
-            pooler_output=pooled_output,
-            past_key_values=encoder_outputs.past_key_values,
-            hidden_states=encoder_outputs.hidden_states,
-            attentions=encoder_outputs.attentions,
-            cross_attentions=encoder_outputs.cross_attentions,
-        )
-
-
-
-class BertLMHeadModel(BertPreTrainedModel):
-
-    _keys_to_ignore_on_load_unexpected = [r"pooler"]
-    _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"]
-
-    def __init__(self, config):
-        super().__init__(config)
-
-        self.bert = BertModel(config, add_pooling_layer=False)
-        self.cls = BertOnlyMLMHead(config)
-
-        self.init_weights()
-
-    def get_output_embeddings(self):
-        return self.cls.predictions.decoder
-
-    def set_output_embeddings(self, new_embeddings):
-        self.cls.predictions.decoder = new_embeddings
-
-    def forward(
-        self,
-        input_ids=None,
-        attention_mask=None,
-        position_ids=None,
-        head_mask=None,
-        inputs_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        labels=None,
-        past_key_values=None,
-        use_cache=None,
-        output_attentions=None,
-        output_hidden_states=None,
-        return_dict=None,
-        return_logits=False,            
-        is_decoder=True,
-        reduction='mean',
-        mode='multimodal', 
-    ):
-        r"""
-        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
-            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
-            the model is configured as a decoder.
-        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
-            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
-            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
-            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
-            ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are
-            ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]``
-        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
-            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
-            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
-            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
-            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
-        use_cache (:obj:`bool`, `optional`):
-            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
-            decoding (see :obj:`past_key_values`).
-        Returns:
-        Example::
-            >>> from transformers import BertTokenizer, BertLMHeadModel, BertConfig
-            >>> import torch
-            >>> tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
-            >>> config = BertConfig.from_pretrained("bert-base-cased")
-            >>> model = BertLMHeadModel.from_pretrained('bert-base-cased', config=config)
-            >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
-            >>> outputs = model(**inputs)
-            >>> prediction_logits = outputs.logits
-        """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-        if labels is not None:
-            use_cache = False
-
-        outputs = self.bert(
-            input_ids,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-            head_mask=head_mask,
-            inputs_embeds=inputs_embeds,
-            encoder_hidden_states=encoder_hidden_states,
-            encoder_attention_mask=encoder_attention_mask,
-            past_key_values=past_key_values,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-            is_decoder=is_decoder,
-            mode=mode,
-        )
-        
-        sequence_output = outputs[0]
-        prediction_scores = self.cls(sequence_output)
-        
-        if return_logits:
-            return prediction_scores[:, :-1, :].contiguous()  
-
-        lm_loss = None
-        if labels is not None:
-            # we are doing next-token prediction; shift prediction scores and input ids by one
-            shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous()
-            labels = labels[:, 1:].contiguous()
-            loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1) 
-            lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
-            if reduction=='none':
-                lm_loss = lm_loss.view(prediction_scores.size(0),-1).sum(1)               
-
-        if not return_dict:
-            output = (prediction_scores,) + outputs[2:]
-            return ((lm_loss,) + output) if lm_loss is not None else output
-
-        return CausalLMOutputWithCrossAttentions(
-            loss=lm_loss,
-            logits=prediction_scores,
-            past_key_values=outputs.past_key_values,
-            hidden_states=outputs.hidden_states,
-            attentions=outputs.attentions,
-            cross_attentions=outputs.cross_attentions,
-        )
-
-    def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, **model_kwargs):
-        input_shape = input_ids.shape
-        # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly
-        if attention_mask is None:
-            attention_mask = input_ids.new_ones(input_shape)
-
-        # cut decoder_input_ids if past is used
-        if past is not None:
-            input_ids = input_ids[:, -1:]
-
-        return {
-            "input_ids": input_ids, 
-            "attention_mask": attention_mask, 
-            "past_key_values": past,
-            "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None),
-            "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None),
-            "is_decoder": True,
-        }
-
-    def _reorder_cache(self, past, beam_idx):
-        reordered_past = ()
-        for layer_past in past:
-            reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),)
-        return reordered_past

extras/BLIP/models/nlvr_encoder.py

@@ -1,843 +0,0 @@
-import math
-import os
-import warnings
-from dataclasses import dataclass
-from typing import Optional, Tuple
-
-import torch
-from torch import Tensor, device, dtype, nn
-import torch.utils.checkpoint
-from torch import nn
-from torch.nn import CrossEntropyLoss
-import torch.nn.functional as F
-
-from transformers.activations import ACT2FN
-from transformers.file_utils import (
-    ModelOutput,
-)
-from transformers.modeling_outputs import (
-    BaseModelOutputWithPastAndCrossAttentions,
-    BaseModelOutputWithPoolingAndCrossAttentions,
-    CausalLMOutputWithCrossAttentions,
-    MaskedLMOutput,
-    MultipleChoiceModelOutput,
-    NextSentencePredictorOutput,
-    QuestionAnsweringModelOutput,
-    SequenceClassifierOutput,
-    TokenClassifierOutput,
-)
-from transformers.modeling_utils import (
-    PreTrainedModel,
-    apply_chunking_to_forward,
-    find_pruneable_heads_and_indices,
-    prune_linear_layer,
-)
-from transformers.utils import logging
-from transformers.models.bert.configuration_bert import BertConfig
-
-
-logger = logging.get_logger(__name__)
-
-
-class BertEmbeddings(nn.Module):
-    """Construct the embeddings from word and position embeddings."""
-
-    def __init__(self, config):
-        super().__init__()
-        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
-        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)
-
-        # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load
-        # any TensorFlow checkpoint file
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)
-
-        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
-        self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)))
-        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
-        
-        self.config = config
-
-    def forward(
-        self, input_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0
-    ):
-        if input_ids is not None:
-            input_shape = input_ids.size()
-        else:
-            input_shape = inputs_embeds.size()[:-1]
-
-        seq_length = input_shape[1]
-
-        if position_ids is None:
-            position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
-
-        if inputs_embeds is None:
-            inputs_embeds = self.word_embeddings(input_ids)
-
-        embeddings = inputs_embeds
-
-        if self.position_embedding_type == "absolute":
-            position_embeddings = self.position_embeddings(position_ids)
-            embeddings += position_embeddings
-        embeddings = self.LayerNorm(embeddings)
-        embeddings = self.dropout(embeddings)
-        return embeddings
-
-
-class BertSelfAttention(nn.Module):
-    def __init__(self, config, is_cross_attention):
-        super().__init__()
-        self.config = config
-        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
-            raise ValueError(
-                "The hidden size (%d) is not a multiple of the number of attention "
-                "heads (%d)" % (config.hidden_size, config.num_attention_heads)
-            )
-        
-        self.num_attention_heads = config.num_attention_heads
-        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
-        self.all_head_size = self.num_attention_heads * self.attention_head_size
-
-        self.query = nn.Linear(config.hidden_size, self.all_head_size)
-        if is_cross_attention:
-            self.key = nn.Linear(config.encoder_width, self.all_head_size)
-            self.value = nn.Linear(config.encoder_width, self.all_head_size)
-        else:
-            self.key = nn.Linear(config.hidden_size, self.all_head_size)
-            self.value = nn.Linear(config.hidden_size, self.all_head_size)
-
-        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
-        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
-        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
-            self.max_position_embeddings = config.max_position_embeddings
-            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)
-        self.save_attention = False   
-            
-    def save_attn_gradients(self, attn_gradients):
-        self.attn_gradients = attn_gradients
-        
-    def get_attn_gradients(self):
-        return self.attn_gradients
-    
-    def save_attention_map(self, attention_map):
-        self.attention_map = attention_map
-        
-    def get_attention_map(self):
-        return self.attention_map
-    
-    def transpose_for_scores(self, x):
-        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
-        x = x.view(*new_x_shape)
-        return x.permute(0, 2, 1, 3)
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_value=None,
-        output_attentions=False,
-    ):
-        mixed_query_layer = self.query(hidden_states)
-
-        # If this is instantiated as a cross-attention module, the keys
-        # and values come from an encoder; the attention mask needs to be
-        # such that the encoder's padding tokens are not attended to.
-        is_cross_attention = encoder_hidden_states is not None
-
-        if is_cross_attention:
-            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
-            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
-            attention_mask = encoder_attention_mask
-        elif past_key_value is not None:
-            key_layer = self.transpose_for_scores(self.key(hidden_states))
-            value_layer = self.transpose_for_scores(self.value(hidden_states))
-            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
-            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
-        else:
-            key_layer = self.transpose_for_scores(self.key(hidden_states))
-            value_layer = self.transpose_for_scores(self.value(hidden_states))
-
-        query_layer = self.transpose_for_scores(mixed_query_layer)
-
-        past_key_value = (key_layer, value_layer)
-
-        # Take the dot product between "query" and "key" to get the raw attention scores.
-        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
-
-        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
-            seq_length = hidden_states.size()[1]
-            position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
-            position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
-            distance = position_ids_l - position_ids_r
-            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
-            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility
-
-            if self.position_embedding_type == "relative_key":
-                relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
-                attention_scores = attention_scores + relative_position_scores
-            elif self.position_embedding_type == "relative_key_query":
-                relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
-                relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
-                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
-
-        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
-        if attention_mask is not None:
-            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
-            attention_scores = attention_scores + attention_mask
-
-        # Normalize the attention scores to probabilities.
-        attention_probs = nn.Softmax(dim=-1)(attention_scores)
-        
-        if is_cross_attention and self.save_attention:
-            self.save_attention_map(attention_probs)
-            attention_probs.register_hook(self.save_attn_gradients)         
-
-        # This is actually dropping out entire tokens to attend to, which might
-        # seem a bit unusual, but is taken from the original Transformer paper.
-        attention_probs_dropped = self.dropout(attention_probs)
-
-        # Mask heads if we want to
-        if head_mask is not None:
-            attention_probs_dropped = attention_probs_dropped * head_mask
-
-        context_layer = torch.matmul(attention_probs_dropped, value_layer)
-
-        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
-        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
-        context_layer = context_layer.view(*new_context_layer_shape)
-
-        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
-
-        outputs = outputs + (past_key_value,)
-        return outputs
-
-
-class BertSelfOutput(nn.Module):
-    def __init__(self, config, twin=False, merge=False):     
-        super().__init__()
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)        
-        if twin:
-            self.dense0 = nn.Linear(config.hidden_size, config.hidden_size)
-            self.dense1 = nn.Linear(config.hidden_size, config.hidden_size)         
-        else:
-            self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        if merge:
-            self.act =  ACT2FN[config.hidden_act]
-            self.merge_layer = nn.Linear(config.hidden_size * 2, config.hidden_size)
-            self.merge = True
-        else:
-            self.merge = False
-
-    def forward(self, hidden_states, input_tensor):
-        if type(hidden_states) == list:
-            hidden_states0 = self.dense0(hidden_states[0])
-            hidden_states1 = self.dense1(hidden_states[1])        
-            if self.merge:  
-                #hidden_states = self.merge_layer(self.act(torch.cat([hidden_states0,hidden_states1],dim=-1)))
-                hidden_states = self.merge_layer(torch.cat([hidden_states0,hidden_states1],dim=-1))
-            else:
-                hidden_states = (hidden_states0+hidden_states1)/2
-        else:    
-            hidden_states = self.dense(hidden_states)
-        hidden_states = self.dropout(hidden_states)
-        hidden_states = self.LayerNorm(hidden_states + input_tensor)
-        return hidden_states
-
-
-class BertAttention(nn.Module):
-    def __init__(self, config, is_cross_attention=False, layer_num=-1):
-        super().__init__()
-        if is_cross_attention:
-            self.self0 = BertSelfAttention(config, is_cross_attention)
-            self.self1 = BertSelfAttention(config, is_cross_attention)
-        else:    
-            self.self = BertSelfAttention(config, is_cross_attention)
-        self.output = BertSelfOutput(config, twin=is_cross_attention, merge=(is_cross_attention and layer_num>=6))
-        self.pruned_heads = set()
-
-    def prune_heads(self, heads):
-        if len(heads) == 0:
-            return
-        heads, index = find_pruneable_heads_and_indices(
-            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
-        )
-
-        # Prune linear layers
-        self.self.query = prune_linear_layer(self.self.query, index)
-        self.self.key = prune_linear_layer(self.self.key, index)
-        self.self.value = prune_linear_layer(self.self.value, index)
-        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)
-
-        # Update hyper params and store pruned heads
-        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
-        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
-        self.pruned_heads = self.pruned_heads.union(heads)
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_value=None,
-        output_attentions=False,
-    ):        
-        if type(encoder_hidden_states)==list:   
-            self_outputs0 = self.self0(
-                hidden_states,
-                attention_mask,
-                head_mask,
-                encoder_hidden_states[0],
-                encoder_attention_mask[0],
-                past_key_value,
-                output_attentions,
-            )
-            self_outputs1 = self.self1(
-                hidden_states,
-                attention_mask,
-                head_mask,
-                encoder_hidden_states[1],
-                encoder_attention_mask[1],
-                past_key_value,
-                output_attentions,
-            )                        
-            attention_output = self.output([self_outputs0[0],self_outputs1[0]], hidden_states)
-    
-            outputs = (attention_output,) + self_outputs0[1:]  # add attentions if we output them
-        else:        
-            self_outputs = self.self(
-                hidden_states,
-                attention_mask,
-                head_mask,
-                encoder_hidden_states,
-                encoder_attention_mask,
-                past_key_value,
-                output_attentions,
-            )
-            attention_output = self.output(self_outputs[0], hidden_states)
-            outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
-        return outputs
-
-
-class BertIntermediate(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
-        if isinstance(config.hidden_act, str):
-            self.intermediate_act_fn = ACT2FN[config.hidden_act]
-        else:
-            self.intermediate_act_fn = config.hidden_act
-
-    def forward(self, hidden_states):
-        hidden_states = self.dense(hidden_states)
-        hidden_states = self.intermediate_act_fn(hidden_states)
-        return hidden_states
-
-
-class BertOutput(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-        self.dropout = nn.Dropout(config.hidden_dropout_prob)
-
-    def forward(self, hidden_states, input_tensor):
-        hidden_states = self.dense(hidden_states)
-        hidden_states = self.dropout(hidden_states)
-        hidden_states = self.LayerNorm(hidden_states + input_tensor)
-        return hidden_states
-
-
-class BertLayer(nn.Module):
-    def __init__(self, config, layer_num):
-        super().__init__()
-        self.config = config
-        self.chunk_size_feed_forward = config.chunk_size_feed_forward
-        self.seq_len_dim = 1
-        self.attention = BertAttention(config)      
-        self.layer_num = layer_num          
-        if self.config.add_cross_attention:
-            self.crossattention = BertAttention(config, is_cross_attention=self.config.add_cross_attention, layer_num=layer_num)
-        self.intermediate = BertIntermediate(config)
-        self.output = BertOutput(config)
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_value=None,
-        output_attentions=False,
-        mode=None,
-    ):
-        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
-        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
-        self_attention_outputs = self.attention(
-            hidden_states,
-            attention_mask,
-            head_mask,
-            output_attentions=output_attentions,
-            past_key_value=self_attn_past_key_value,
-        )
-        attention_output = self_attention_outputs[0]
-
-        outputs = self_attention_outputs[1:-1]
-        present_key_value = self_attention_outputs[-1]
-
-        if mode=='multimodal':
-            assert encoder_hidden_states is not None, "encoder_hidden_states must be given for cross-attention layers"
-            cross_attention_outputs = self.crossattention(
-                attention_output,
-                attention_mask,
-                head_mask,
-                encoder_hidden_states,
-                encoder_attention_mask,
-                output_attentions=output_attentions,
-            )
-            attention_output = cross_attention_outputs[0]
-            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights                               
-        layer_output = apply_chunking_to_forward(
-            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
-        )
-        outputs = (layer_output,) + outputs
-
-        outputs = outputs + (present_key_value,)
-
-        return outputs
-
-    def feed_forward_chunk(self, attention_output):
-        intermediate_output = self.intermediate(attention_output)
-        layer_output = self.output(intermediate_output, attention_output)
-        return layer_output
-
-
-class BertEncoder(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.config = config
-        self.layer = nn.ModuleList([BertLayer(config,i) for i in range(config.num_hidden_layers)])
-        self.gradient_checkpointing = False
-
-    def forward(
-        self,
-        hidden_states,
-        attention_mask=None,
-        head_mask=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_values=None,
-        use_cache=None,
-        output_attentions=False,
-        output_hidden_states=False,
-        return_dict=True,
-        mode='multimodal',
-    ):
-        all_hidden_states = () if output_hidden_states else None
-        all_self_attentions = () if output_attentions else None
-        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
-
-        next_decoder_cache = () if use_cache else None
-               
-        for i in range(self.config.num_hidden_layers):
-            layer_module = self.layer[i]
-            if output_hidden_states:
-                all_hidden_states = all_hidden_states + (hidden_states,)
-
-            layer_head_mask = head_mask[i] if head_mask is not None else None
-            past_key_value = past_key_values[i] if past_key_values is not None else None
-
-            if self.gradient_checkpointing and self.training:
-
-                if use_cache:
-                    logger.warn(
-                        "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
-                    )
-                    use_cache = False
-
-                def create_custom_forward(module):
-                    def custom_forward(*inputs):
-                        return module(*inputs, past_key_value, output_attentions)
-
-                    return custom_forward
-
-                layer_outputs = torch.utils.checkpoint.checkpoint(
-                    create_custom_forward(layer_module),
-                    hidden_states,
-                    attention_mask,
-                    layer_head_mask,
-                    encoder_hidden_states,
-                    encoder_attention_mask,
-                    mode=mode,
-                )
-            else:
-                layer_outputs = layer_module(
-                    hidden_states,
-                    attention_mask,
-                    layer_head_mask,
-                    encoder_hidden_states,
-                    encoder_attention_mask,
-                    past_key_value,
-                    output_attentions,
-                    mode=mode,
-                )
-
-            hidden_states = layer_outputs[0]
-            if use_cache:
-                next_decoder_cache += (layer_outputs[-1],)
-            if output_attentions:
-                all_self_attentions = all_self_attentions + (layer_outputs[1],)
-
-        if output_hidden_states:
-            all_hidden_states = all_hidden_states + (hidden_states,)
-
-        if not return_dict:
-            return tuple(
-                v
-                for v in [
-                    hidden_states,
-                    next_decoder_cache,
-                    all_hidden_states,
-                    all_self_attentions,
-                    all_cross_attentions,
-                ]
-                if v is not None
-            )
-        return BaseModelOutputWithPastAndCrossAttentions(
-            last_hidden_state=hidden_states,
-            past_key_values=next_decoder_cache,
-            hidden_states=all_hidden_states,
-            attentions=all_self_attentions,
-            cross_attentions=all_cross_attentions,
-        )
-
-
-class BertPooler(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        self.activation = nn.Tanh()
-
-    def forward(self, hidden_states):
-        # We "pool" the model by simply taking the hidden state corresponding
-        # to the first token.
-        first_token_tensor = hidden_states[:, 0]
-        pooled_output = self.dense(first_token_tensor)
-        pooled_output = self.activation(pooled_output)
-        return pooled_output
-
-
-class BertPredictionHeadTransform(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
-        if isinstance(config.hidden_act, str):
-            self.transform_act_fn = ACT2FN[config.hidden_act]
-        else:
-            self.transform_act_fn = config.hidden_act
-        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
-
-    def forward(self, hidden_states):
-        hidden_states = self.dense(hidden_states)
-        hidden_states = self.transform_act_fn(hidden_states)
-        hidden_states = self.LayerNorm(hidden_states)
-        return hidden_states
-
-
-class BertLMPredictionHead(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.transform = BertPredictionHeadTransform(config)
-
-        # The output weights are the same as the input embeddings, but there is
-        # an output-only bias for each token.
-        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
-
-        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
-
-        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
-        self.decoder.bias = self.bias
-
-    def forward(self, hidden_states):
-        hidden_states = self.transform(hidden_states)
-        hidden_states = self.decoder(hidden_states)
-        return hidden_states
-
-
-class BertOnlyMLMHead(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.predictions = BertLMPredictionHead(config)
-
-    def forward(self, sequence_output):
-        prediction_scores = self.predictions(sequence_output)
-        return prediction_scores
-
-
-class BertPreTrainedModel(PreTrainedModel):
-    """
-    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
-    models.
-    """
-
-    config_class = BertConfig
-    base_model_prefix = "bert"
-    _keys_to_ignore_on_load_missing = [r"position_ids"]
-
-    def _init_weights(self, module):
-        """ Initialize the weights """
-        if isinstance(module, (nn.Linear, nn.Embedding)):
-            # Slightly different from the TF version which uses truncated_normal for initialization
-            # cf https://github.com/pytorch/pytorch/pull/5617
-            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
-        elif isinstance(module, nn.LayerNorm):
-            module.bias.data.zero_()
-            module.weight.data.fill_(1.0)
-        if isinstance(module, nn.Linear) and module.bias is not None:
-            module.bias.data.zero_()
-
-
-class BertModel(BertPreTrainedModel):
-    """
-    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
-    cross-attention is added between the self-attention layers, following the architecture described in `Attention is
-    all you need <https://arxiv.org/abs/1706.03762>`__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
-    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
-    argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an
-    input to the forward pass.
-    """
-
-    def __init__(self, config, add_pooling_layer=True):
-        super().__init__(config)
-        self.config = config
-
-        self.embeddings = BertEmbeddings(config)
-        
-        self.encoder = BertEncoder(config)
-
-        self.pooler = BertPooler(config) if add_pooling_layer else None
-
-        self.init_weights()
- 
-
-    def get_input_embeddings(self):
-        return self.embeddings.word_embeddings
-
-    def set_input_embeddings(self, value):
-        self.embeddings.word_embeddings = value
-
-    def _prune_heads(self, heads_to_prune):
-        """
-        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
-        class PreTrainedModel
-        """
-        for layer, heads in heads_to_prune.items():
-            self.encoder.layer[layer].attention.prune_heads(heads)
-
-    
-    def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool) -> Tensor:
-        """
-        Makes broadcastable attention and causal masks so that future and masked tokens are ignored.
-
-        Arguments:
-            attention_mask (:obj:`torch.Tensor`):
-                Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
-            input_shape (:obj:`Tuple[int]`):
-                The shape of the input to the model.
-            device: (:obj:`torch.device`):
-                The device of the input to the model.
-
-        Returns:
-            :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`.
-        """
-        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
-        # ourselves in which case we just need to make it broadcastable to all heads.
-        if attention_mask.dim() == 3:
-            extended_attention_mask = attention_mask[:, None, :, :]
-        elif attention_mask.dim() == 2:
-            # Provided a padding mask of dimensions [batch_size, seq_length]
-            # - if the model is a decoder, apply a causal mask in addition to the padding mask
-            # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
-            if is_decoder:
-                batch_size, seq_length = input_shape
-
-                seq_ids = torch.arange(seq_length, device=device)
-                causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
-                # in case past_key_values are used we need to add a prefix ones mask to the causal mask
-                # causal and attention masks must have same type with pytorch version < 1.3
-                causal_mask = causal_mask.to(attention_mask.dtype)
-   
-                if causal_mask.shape[1] < attention_mask.shape[1]:
-                    prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1]
-                    causal_mask = torch.cat(
-                        [
-                            torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype),
-                            causal_mask,
-                        ],
-                        axis=-1,
-                    )                     
-
-                extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
-            else:
-                extended_attention_mask = attention_mask[:, None, None, :]
-        else:
-            raise ValueError(
-                "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(
-                    input_shape, attention_mask.shape
-                )
-            )
-
-        # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
-        # masked positions, this operation will create a tensor which is 0.0 for
-        # positions we want to attend and -10000.0 for masked positions.
-        # Since we are adding it to the raw scores before the softmax, this is
-        # effectively the same as removing these entirely.
-        extended_attention_mask = extended_attention_mask.to(dtype=self.dtype)  # fp16 compatibility
-        extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
-        return extended_attention_mask
-    
-    def forward(
-        self,
-        input_ids=None,
-        attention_mask=None,
-        position_ids=None,
-        head_mask=None,
-        inputs_embeds=None,
-        encoder_embeds=None,
-        encoder_hidden_states=None,
-        encoder_attention_mask=None,
-        past_key_values=None,
-        use_cache=None,
-        output_attentions=None,
-        output_hidden_states=None,
-        return_dict=None,
-        is_decoder=False,
-        mode='multimodal',
-    ):
-        r"""
-        encoder_hidden_states  (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`):
-            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
-            the model is configured as a decoder.
-        encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
-            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
-            the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``:
-            - 1 for tokens that are **not masked**,
-            - 0 for tokens that are **masked**.
-        past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
-            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
-            If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids`
-            (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)`
-            instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`.
-        use_cache (:obj:`bool`, `optional`):
-            If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up
-            decoding (see :obj:`past_key_values`).
-        """
-        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        if is_decoder:
-            use_cache = use_cache if use_cache is not None else self.config.use_cache
-        else:
-            use_cache = False
-
-        if input_ids is not None and inputs_embeds is not None:
-            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
-        elif input_ids is not None:
-            input_shape = input_ids.size()
-            batch_size, seq_length = input_shape
-            device = input_ids.device
-        elif inputs_embeds is not None:
-            input_shape = inputs_embeds.size()[:-1]
-            batch_size, seq_length = input_shape
-            device = inputs_embeds.device
-        elif encoder_embeds is not None:    
-            input_shape = encoder_embeds.size()[:-1]
-            batch_size, seq_length = input_shape 
-            device = encoder_embeds.device
-        else:
-            raise ValueError("You have to specify either input_ids or inputs_embeds or encoder_embeds")
-
-        # past_key_values_length
-        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0
-
-        if attention_mask is None:
-            attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device)
-            
-        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
-        # ourselves in which case we just need to make it broadcastable to all heads.
-        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, 
-                                                                                 device, is_decoder)
-
-        # If a 2D or 3D attention mask is provided for the cross-attention
-        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
-        if encoder_hidden_states is not None:
-            if type(encoder_hidden_states) == list:
-                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size()
-            else:
-                encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
-            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
-            
-            if type(encoder_attention_mask) == list:
-                encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask]
-            elif encoder_attention_mask is None:
-                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
-                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
-            else:    
-                encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
-        else:
-            encoder_extended_attention_mask = None
-
-        # Prepare head mask if needed
-        # 1.0 in head_mask indicate we keep the head
-        # attention_probs has shape bsz x n_heads x N x N
-        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
-        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
-        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
-        
-        if encoder_embeds is None:
-            embedding_output = self.embeddings(
-                input_ids=input_ids,
-                position_ids=position_ids,
-                inputs_embeds=inputs_embeds,
-                past_key_values_length=past_key_values_length,
-            )
-        else:
-            embedding_output = encoder_embeds
-            
-        encoder_outputs = self.encoder(
-            embedding_output,
-            attention_mask=extended_attention_mask,
-            head_mask=head_mask,
-            encoder_hidden_states=encoder_hidden_states,
-            encoder_attention_mask=encoder_extended_attention_mask,
-            past_key_values=past_key_values,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-            mode=mode,
-        )
-        sequence_output = encoder_outputs[0]
-        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
-
-        if not return_dict:
-            return (sequence_output, pooled_output) + encoder_outputs[1:]
-
-        return BaseModelOutputWithPoolingAndCrossAttentions(
-            last_hidden_state=sequence_output,
-            pooler_output=pooled_output,
-            past_key_values=encoder_outputs.past_key_values,
-            hidden_states=encoder_outputs.hidden_states,
-            attentions=encoder_outputs.attentions,
-            cross_attentions=encoder_outputs.cross_attentions,
-        )
-

extras/BLIP/models/vit.py

@@ -1,308 +0,0 @@
-'''
- * Copyright (c) 2022, salesforce.com, inc.
- * All rights reserved.
- * SPDX-License-Identifier: BSD-3-Clause
- * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
- * By Junnan Li
- * Based on timm code base
- * https://github.com/rwightman/pytorch-image-models/tree/master/timm
-'''
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from functools import partial
-
-from timm.models.vision_transformer import _cfg, PatchEmbed
-from timm.models.registry import register_model
-from timm.models.layers import trunc_normal_, DropPath
-from timm.models.helpers import named_apply, adapt_input_conv
-
-
-def checkpoint_wrapper(x):
-    return x
-
-
-class Mlp(nn.Module):
-    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
-    """
-    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):
-        super().__init__()
-        out_features = out_features or in_features
-        hidden_features = hidden_features or in_features
-        self.fc1 = nn.Linear(in_features, hidden_features)
-        self.act = act_layer()
-        self.fc2 = nn.Linear(hidden_features, out_features)
-        self.drop = nn.Dropout(drop)
-
-    def forward(self, x):
-        x = self.fc1(x)
-        x = self.act(x)
-        x = self.drop(x)
-        x = self.fc2(x)
-        x = self.drop(x)
-        return x
-
-
-class Attention(nn.Module):
-    def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.):
-        super().__init__()
-        self.num_heads = num_heads
-        head_dim = dim // num_heads
-        # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights
-        self.scale = qk_scale or head_dim ** -0.5
-        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
-        self.attn_drop = nn.Dropout(attn_drop)
-        self.proj = nn.Linear(dim, dim)
-        self.proj_drop = nn.Dropout(proj_drop)
-        self.attn_gradients = None
-        self.attention_map = None
-        
-    def save_attn_gradients(self, attn_gradients):
-        self.attn_gradients = attn_gradients
-        
-    def get_attn_gradients(self):
-        return self.attn_gradients
-    
-    def save_attention_map(self, attention_map):
-        self.attention_map = attention_map
-        
-    def get_attention_map(self):
-        return self.attention_map
-    
-    def forward(self, x, register_hook=False):
-        B, N, C = x.shape
-        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
-        q, k, v = qkv[0], qkv[1], qkv[2]   # make torchscript happy (cannot use tensor as tuple)
-
-        attn = (q @ k.transpose(-2, -1)) * self.scale
-        attn = attn.softmax(dim=-1)
-        attn = self.attn_drop(attn)
-                
-        if register_hook:
-            self.save_attention_map(attn)
-            attn.register_hook(self.save_attn_gradients)        
-
-        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
-        x = self.proj(x)
-        x = self.proj_drop(x)
-        return x
-
-
-class Block(nn.Module):
-
-    def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0.,
-                 drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, use_grad_checkpointing=False):
-        super().__init__()
-        self.norm1 = norm_layer(dim)
-        self.attn = Attention(
-            dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
-        # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here
-        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
-        self.norm2 = norm_layer(dim)
-        mlp_hidden_dim = int(dim * mlp_ratio)
-        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)
-
-        if use_grad_checkpointing:
-            self.attn = checkpoint_wrapper(self.attn)
-            self.mlp = checkpoint_wrapper(self.mlp)
-
-    def forward(self, x, register_hook=False):
-        x = x + self.drop_path(self.attn(self.norm1(x), register_hook=register_hook))
-        x = x + self.drop_path(self.mlp(self.norm2(x)))
-        return x
-
-    
-class VisionTransformer(nn.Module):
-    """ Vision Transformer
-    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`  -
-        https://arxiv.org/abs/2010.11929
-    """
-    def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12,
-                 num_heads=12, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None,
-                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=None, 
-                 use_grad_checkpointing=False, ckpt_layer=0):
-        """
-        Args:
-            img_size (int, tuple): input image size
-            patch_size (int, tuple): patch size
-            in_chans (int): number of input channels
-            num_classes (int): number of classes for classification head
-            embed_dim (int): embedding dimension
-            depth (int): depth of transformer
-            num_heads (int): number of attention heads
-            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
-            qkv_bias (bool): enable bias for qkv if True
-            qk_scale (float): override default qk scale of head_dim ** -0.5 if set
-            representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set
-            drop_rate (float): dropout rate
-            attn_drop_rate (float): attention dropout rate
-            drop_path_rate (float): stochastic depth rate
-            norm_layer: (nn.Module): normalization layer
-        """
-        super().__init__()
-        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
-        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
-
-        self.patch_embed = PatchEmbed(
-            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
-
-        num_patches = self.patch_embed.num_patches
-
-        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
-        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))
-        self.pos_drop = nn.Dropout(p=drop_rate)
-
-        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
-        self.blocks = nn.ModuleList([
-            Block(
-                dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale,
-                drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer,
-                use_grad_checkpointing=(use_grad_checkpointing and i>=depth-ckpt_layer)
-            )
-            for i in range(depth)])
-        self.norm = norm_layer(embed_dim)
-
-        trunc_normal_(self.pos_embed, std=.02)
-        trunc_normal_(self.cls_token, std=.02)
-        self.apply(self._init_weights)
-
-    def _init_weights(self, m):
-        if isinstance(m, nn.Linear):
-            trunc_normal_(m.weight, std=.02)
-            if isinstance(m, nn.Linear) and m.bias is not None:
-                nn.init.constant_(m.bias, 0)
-        elif isinstance(m, nn.LayerNorm):
-            nn.init.constant_(m.bias, 0)
-            nn.init.constant_(m.weight, 1.0)
-
-    @torch.jit.ignore
-    def no_weight_decay(self):
-        return {'pos_embed', 'cls_token'}
-
-    def forward(self, x, register_blk=-1):
-        B = x.shape[0]
-        x = self.patch_embed(x)
-
-        cls_tokens = self.cls_token.expand(B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
-        x = torch.cat((cls_tokens, x), dim=1)
-  
-        x = x + self.pos_embed[:,:x.size(1),:]
-        x = self.pos_drop(x)
-
-        for i,blk in enumerate(self.blocks):
-            x = blk(x, register_blk==i)
-        x = self.norm(x)
-        
-        return x
-
-    @torch.jit.ignore()
-    def load_pretrained(self, checkpoint_path, prefix=''):
-        _load_weights(self, checkpoint_path, prefix)
-        
-
-@torch.no_grad()
-def _load_weights(model: VisionTransformer, checkpoint_path: str, prefix: str = ''):
-    """ Load weights from .npz checkpoints for official Google Brain Flax implementation
-    """
-    import numpy as np
-
-    def _n2p(w, t=True):
-        if w.ndim == 4 and w.shape[0] == w.shape[1] == w.shape[2] == 1:
-            w = w.flatten()
-        if t:
-            if w.ndim == 4:
-                w = w.transpose([3, 2, 0, 1])
-            elif w.ndim == 3:
-                w = w.transpose([2, 0, 1])
-            elif w.ndim == 2:
-                w = w.transpose([1, 0])
-        return torch.from_numpy(w)
-
-    w = np.load(checkpoint_path)
-    if not prefix and 'opt/target/embedding/kernel' in w:
-        prefix = 'opt/target/'
-
-    if hasattr(model.patch_embed, 'backbone'):
-        # hybrid
-        backbone = model.patch_embed.backbone
-        stem_only = not hasattr(backbone, 'stem')
-        stem = backbone if stem_only else backbone.stem
-        stem.conv.weight.copy_(adapt_input_conv(stem.conv.weight.shape[1], _n2p(w[f'{prefix}conv_root/kernel'])))
-        stem.norm.weight.copy_(_n2p(w[f'{prefix}gn_root/scale']))
-        stem.norm.bias.copy_(_n2p(w[f'{prefix}gn_root/bias']))
-        if not stem_only:
-            for i, stage in enumerate(backbone.stages):
-                for j, block in enumerate(stage.blocks):
-                    bp = f'{prefix}block{i + 1}/unit{j + 1}/'
-                    for r in range(3):
-                        getattr(block, f'conv{r + 1}').weight.copy_(_n2p(w[f'{bp}conv{r + 1}/kernel']))
-                        getattr(block, f'norm{r + 1}').weight.copy_(_n2p(w[f'{bp}gn{r + 1}/scale']))
-                        getattr(block, f'norm{r + 1}').bias.copy_(_n2p(w[f'{bp}gn{r + 1}/bias']))
-                    if block.downsample is not None:
-                        block.downsample.conv.weight.copy_(_n2p(w[f'{bp}conv_proj/kernel']))
-                        block.downsample.norm.weight.copy_(_n2p(w[f'{bp}gn_proj/scale']))
-                        block.downsample.norm.bias.copy_(_n2p(w[f'{bp}gn_proj/bias']))
-        embed_conv_w = _n2p(w[f'{prefix}embedding/kernel'])
-    else:
-        embed_conv_w = adapt_input_conv(
-            model.patch_embed.proj.weight.shape[1], _n2p(w[f'{prefix}embedding/kernel']))
-    model.patch_embed.proj.weight.copy_(embed_conv_w)
-    model.patch_embed.proj.bias.copy_(_n2p(w[f'{prefix}embedding/bias']))
-    model.cls_token.copy_(_n2p(w[f'{prefix}cls'], t=False))
-    pos_embed_w = _n2p(w[f'{prefix}Transformer/posembed_input/pos_embedding'], t=False)
-    if pos_embed_w.shape != model.pos_embed.shape:
-        pos_embed_w = resize_pos_embed(  # resize pos embedding when different size from pretrained weights
-            pos_embed_w, model.pos_embed, getattr(model, 'num_tokens', 1), model.patch_embed.grid_size)
-    model.pos_embed.copy_(pos_embed_w)
-    model.norm.weight.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/scale']))
-    model.norm.bias.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/bias']))
-#     if isinstance(model.head, nn.Linear) and model.head.bias.shape[0] == w[f'{prefix}head/bias'].shape[-1]:
-#         model.head.weight.copy_(_n2p(w[f'{prefix}head/kernel']))
-#         model.head.bias.copy_(_n2p(w[f'{prefix}head/bias']))
-#     if isinstance(getattr(model.pre_logits, 'fc', None), nn.Linear) and f'{prefix}pre_logits/bias' in w:
-#         model.pre_logits.fc.weight.copy_(_n2p(w[f'{prefix}pre_logits/kernel']))
-#         model.pre_logits.fc.bias.copy_(_n2p(w[f'{prefix}pre_logits/bias']))
-    for i, block in enumerate(model.blocks.children()):
-        block_prefix = f'{prefix}Transformer/encoderblock_{i}/'
-        mha_prefix = block_prefix + 'MultiHeadDotProductAttention_1/'
-        block.norm1.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/scale']))
-        block.norm1.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/bias']))
-        block.attn.qkv.weight.copy_(torch.cat([
-            _n2p(w[f'{mha_prefix}{n}/kernel'], t=False).flatten(1).T for n in ('query', 'key', 'value')]))
-        block.attn.qkv.bias.copy_(torch.cat([
-            _n2p(w[f'{mha_prefix}{n}/bias'], t=False).reshape(-1) for n in ('query', 'key', 'value')]))
-        block.attn.proj.weight.copy_(_n2p(w[f'{mha_prefix}out/kernel']).flatten(1))
-        block.attn.proj.bias.copy_(_n2p(w[f'{mha_prefix}out/bias']))
-        for r in range(2):
-            getattr(block.mlp, f'fc{r + 1}').weight.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/kernel']))
-            getattr(block.mlp, f'fc{r + 1}').bias.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/bias']))
-        block.norm2.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/scale']))
-        block.norm2.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/bias']))
-
-            
-def interpolate_pos_embed(pos_embed_checkpoint, visual_encoder):        
-    # interpolate position embedding
-    embedding_size = pos_embed_checkpoint.shape[-1]
-    num_patches = visual_encoder.patch_embed.num_patches
-    num_extra_tokens = visual_encoder.pos_embed.shape[-2] - num_patches
-    # height (== width) for the checkpoint position embedding
-    orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
-    # height (== width) for the new position embedding
-    new_size = int(num_patches ** 0.5)
-
-    if orig_size!=new_size:
-        # class_token and dist_token are kept unchanged
-        extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
-        # only the position tokens are interpolated
-        pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
-        pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
-        pos_tokens = torch.nn.functional.interpolate(
-            pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
-        pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
-        new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
-        print('reshape position embedding from %d to %d'%(orig_size ** 2,new_size ** 2))
-        
-        return new_pos_embed    
-    else:
-        return pos_embed_checkpoint

extras/expansion.py

@@ -1,129 +0,0 @@
-# Fooocus GPT2 Expansion
-# Algorithm created by Lvmin Zhang at 2023, Stanford
-# If used inside Fooocus, any use is permitted.
-# If used outside Fooocus, only non-commercial use is permitted (CC-By NC 4.0).
-# This applies to the word list, vocab, model, and algorithm.
-
-
-import os
-import torch
-import math
-import ldm_patched.modules.model_management as model_management
-
-from transformers.generation.logits_process import LogitsProcessorList
-from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed
-from modules.config import path_fooocus_expansion
-from ldm_patched.modules.model_patcher import ModelPatcher
-
-
-# limitation of np.random.seed(), called from transformers.set_seed()
-SEED_LIMIT_NUMPY = 2**32
-neg_inf = - 8192.0
-
-
-def safe_str(x):
-    x = str(x)
-    for _ in range(16):
-        x = x.replace('  ', ' ')
-    return x.strip(",. \r\n")
-
-
-def remove_pattern(x, pattern):
-    for p in pattern:
-        x = x.replace(p, '')
-    return x
-
-
-class FooocusExpansion:
-    def __init__(self):
-        self.tokenizer = AutoTokenizer.from_pretrained(path_fooocus_expansion)
-
-        positive_words = open(os.path.join(path_fooocus_expansion, 'positive.txt'),
-                              encoding='utf-8').read().splitlines()
-        positive_words = ['Ġ' + x.lower() for x in positive_words if x != '']
-
-        self.logits_bias = torch.zeros((1, len(self.tokenizer.vocab)), dtype=torch.float32) + neg_inf
-
-        debug_list = []
-        for k, v in self.tokenizer.vocab.items():
-            if k in positive_words:
-                self.logits_bias[0, v] = 0
-                debug_list.append(k[1:])
-
-        print(f'Fooocus V2 Expansion: Vocab with {len(debug_list)} words.')
-
-        # debug_list = '\n'.join(sorted(debug_list))
-        # print(debug_list)
-
-        # t11 = self.tokenizer(',', return_tensors="np")
-        # t198 = self.tokenizer('\n', return_tensors="np")
-        # eos = self.tokenizer.eos_token_id
-
-        self.model = AutoModelForCausalLM.from_pretrained(path_fooocus_expansion)
-        self.model.eval()
-
-        load_device = model_management.text_encoder_device()
-        offload_device = model_management.text_encoder_offload_device()
-
-        # MPS hack
-        if model_management.is_device_mps(load_device):
-            load_device = torch.device('cpu')
-            offload_device = torch.device('cpu')
-
-        use_fp16 = model_management.should_use_fp16(device=load_device)
-
-        if use_fp16:
-            self.model.half()
-
-        self.patcher = ModelPatcher(self.model, load_device=load_device, offload_device=offload_device)
-        print(f'Fooocus Expansion engine loaded for {load_device}, use_fp16 = {use_fp16}.')
-
-    @torch.no_grad()
-    @torch.inference_mode()
-    def logits_processor(self, input_ids, scores):
-        assert scores.ndim == 2 and scores.shape[0] == 1
-        self.logits_bias = self.logits_bias.to(scores)
-
-        bias = self.logits_bias.clone()
-        bias[0, input_ids[0].to(bias.device).long()] = neg_inf
-        bias[0, 11] = 0
-
-        return scores + bias
-
-    @torch.no_grad()
-    @torch.inference_mode()
-    def __call__(self, prompt, seed):
-        if prompt == '':
-            return ''
-
-        if self.patcher.current_device != self.patcher.load_device:
-            print('Fooocus Expansion loaded by itself.')
-            model_management.load_model_gpu(self.patcher)
-
-        seed = int(seed) % SEED_LIMIT_NUMPY
-        set_seed(seed)
-        prompt = safe_str(prompt) + ','
-
-        tokenized_kwargs = self.tokenizer(prompt, return_tensors="pt")
-        tokenized_kwargs.data['input_ids'] = tokenized_kwargs.data['input_ids'].to(self.patcher.load_device)
-        tokenized_kwargs.data['attention_mask'] = tokenized_kwargs.data['attention_mask'].to(self.patcher.load_device)
-
-        current_token_length = int(tokenized_kwargs.data['input_ids'].shape[1])
-        max_token_length = 75 * int(math.ceil(float(current_token_length) / 75.0))
-        max_new_tokens = max_token_length - current_token_length
-
-        if max_new_tokens == 0:
-            return prompt[:-1]
-
-        # https://huggingface.co/blog/introducing-csearch
-        # https://huggingface.co/docs/transformers/generation_strategies
-        features = self.model.generate(**tokenized_kwargs,
-                                       top_k=100,
-                                       max_new_tokens=max_new_tokens,
-                                       do_sample=True,
-                                       logits_processor=LogitsProcessorList([self.logits_processor]))
-
-        response = self.tokenizer.batch_decode(features, skip_special_tokens=True)
-        result = safe_str(response[0])
-
-        return result

extras/face_crop.py

@@ -1,50 +0,0 @@
-import cv2
-import numpy as np
-import modules.config
-
-
-faceRestoreHelper = None
-
-
-def align_warp_face(self, landmark, border_mode='constant'):
-    affine_matrix = cv2.estimateAffinePartial2D(landmark, self.face_template, method=cv2.LMEDS)[0]
-    self.affine_matrices.append(affine_matrix)
-    if border_mode == 'constant':
-        border_mode = cv2.BORDER_CONSTANT
-    elif border_mode == 'reflect101':
-        border_mode = cv2.BORDER_REFLECT101
-    elif border_mode == 'reflect':
-        border_mode = cv2.BORDER_REFLECT
-    input_img = self.input_img
-    cropped_face = cv2.warpAffine(input_img, affine_matrix, self.face_size,
-                                  borderMode=border_mode, borderValue=(135, 133, 132))
-    return cropped_face
-
-
-def crop_image(img_rgb):
-    global faceRestoreHelper
-    
-    if faceRestoreHelper is None:
-        from extras.facexlib.utils.face_restoration_helper import FaceRestoreHelper
-        faceRestoreHelper = FaceRestoreHelper(
-            upscale_factor=1,
-            model_rootpath=modules.config.path_controlnet,
-            device='cpu'  # use cpu is safer since we are out of memory management
-        )
-
-    faceRestoreHelper.clean_all()
-    faceRestoreHelper.read_image(np.ascontiguousarray(img_rgb[:, :, ::-1].copy()))
-    faceRestoreHelper.get_face_landmarks_5()
-
-    landmarks = faceRestoreHelper.all_landmarks_5
-    # landmarks are already sorted with confidence.
-
-    if len(landmarks) == 0:
-        print('No face detected')
-        return img_rgb
-    else:
-        print(f'Detected {len(landmarks)} faces')
-
-    result = align_warp_face(faceRestoreHelper, landmarks[0])
-
-    return np.ascontiguousarray(result[:, :, ::-1].copy())

extras/facexlib/detection/__init__.py

@@ -1,31 +0,0 @@
-import torch
-from copy import deepcopy
-
-from extras.facexlib.utils import load_file_from_url
-from .retinaface import RetinaFace
-
-
-def init_detection_model(model_name, half=False, device='cuda', model_rootpath=None):
-    if model_name == 'retinaface_resnet50':
-        model = RetinaFace(network_name='resnet50', half=half, device=device)
-        model_url = 'https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_Resnet50_Final.pth'
-    elif model_name == 'retinaface_mobile0.25':
-        model = RetinaFace(network_name='mobile0.25', half=half, device=device)
-        model_url = 'https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_mobilenet0.25_Final.pth'
-    else:
-        raise NotImplementedError(f'{model_name} is not implemented.')
-
-    model_path = load_file_from_url(
-        url=model_url, model_dir='facexlib/weights', progress=True, file_name=None, save_dir=model_rootpath)
-
-    # TODO: clean pretrained model
-    load_net = torch.load(model_path, map_location=lambda storage, loc: storage)
-    # remove unnecessary 'module.'
-    for k, v in deepcopy(load_net).items():
-        if k.startswith('module.'):
-            load_net[k[7:]] = v
-            load_net.pop(k)
-    model.load_state_dict(load_net, strict=True)
-    model.eval()
-    model = model.to(device)
-    return model

extras/facexlib/detection/align_trans.py

@@ -1,219 +0,0 @@
-import cv2
-import numpy as np
-
-from .matlab_cp2tform import get_similarity_transform_for_cv2
-
-# reference facial points, a list of coordinates (x,y)
-REFERENCE_FACIAL_POINTS = [[30.29459953, 51.69630051], [65.53179932, 51.50139999], [48.02519989, 71.73660278],
-                           [33.54930115, 92.3655014], [62.72990036, 92.20410156]]
-
-DEFAULT_CROP_SIZE = (96, 112)
-
-
-class FaceWarpException(Exception):
-
-    def __str__(self):
-        return 'In File {}:{}'.format(__file__, super.__str__(self))
-
-
-def get_reference_facial_points(output_size=None, inner_padding_factor=0.0, outer_padding=(0, 0), default_square=False):
-    """
-    Function:
-    ----------
-        get reference 5 key points according to crop settings:
-        0. Set default crop_size:
-            if default_square:
-                crop_size = (112, 112)
-            else:
-                crop_size = (96, 112)
-        1. Pad the crop_size by inner_padding_factor in each side;
-        2. Resize crop_size into (output_size - outer_padding*2),
-            pad into output_size with outer_padding;
-        3. Output reference_5point;
-    Parameters:
-    ----------
-        @output_size: (w, h) or None
-            size of aligned face image
-        @inner_padding_factor: (w_factor, h_factor)
-            padding factor for inner (w, h)
-        @outer_padding: (w_pad, h_pad)
-            each row is a pair of coordinates (x, y)
-        @default_square: True or False
-            if True:
-                default crop_size = (112, 112)
-            else:
-                default crop_size = (96, 112);
-        !!! make sure, if output_size is not None:
-                (output_size - outer_padding)
-                = some_scale * (default crop_size * (1.0 +
-                inner_padding_factor))
-    Returns:
-    ----------
-        @reference_5point: 5x2 np.array
-            each row is a pair of transformed coordinates (x, y)
-    """
-
-    tmp_5pts = np.array(REFERENCE_FACIAL_POINTS)
-    tmp_crop_size = np.array(DEFAULT_CROP_SIZE)
-
-    # 0) make the inner region a square
-    if default_square:
-        size_diff = max(tmp_crop_size) - tmp_crop_size
-        tmp_5pts += size_diff / 2
-        tmp_crop_size += size_diff
-
-    if (output_size and output_size[0] == tmp_crop_size[0] and output_size[1] == tmp_crop_size[1]):
-
-        return tmp_5pts
-
-    if (inner_padding_factor == 0 and outer_padding == (0, 0)):
-        if output_size is None:
-            return tmp_5pts
-        else:
-            raise FaceWarpException('No paddings to do, output_size must be None or {}'.format(tmp_crop_size))
-
-    # check output size
-    if not (0 <= inner_padding_factor <= 1.0):
-        raise FaceWarpException('Not (0 <= inner_padding_factor <= 1.0)')
-
-    if ((inner_padding_factor > 0 or outer_padding[0] > 0 or outer_padding[1] > 0) and output_size is None):
-        output_size = tmp_crop_size * \
-            (1 + inner_padding_factor * 2).astype(np.int32)
-        output_size += np.array(outer_padding)
-    if not (outer_padding[0] < output_size[0] and outer_padding[1] < output_size[1]):
-        raise FaceWarpException('Not (outer_padding[0] < output_size[0] and outer_padding[1] < output_size[1])')
-
-    # 1) pad the inner region according inner_padding_factor
-    if inner_padding_factor > 0:
-        size_diff = tmp_crop_size * inner_padding_factor * 2
-        tmp_5pts += size_diff / 2
-        tmp_crop_size += np.round(size_diff).astype(np.int32)
-
-    # 2) resize the padded inner region
-    size_bf_outer_pad = np.array(output_size) - np.array(outer_padding) * 2
-
-    if size_bf_outer_pad[0] * tmp_crop_size[1] != size_bf_outer_pad[1] * tmp_crop_size[0]:
-        raise FaceWarpException('Must have (output_size - outer_padding)'
-                                '= some_scale * (crop_size * (1.0 + inner_padding_factor)')
-
-    scale_factor = size_bf_outer_pad[0].astype(np.float32) / tmp_crop_size[0]
-    tmp_5pts = tmp_5pts * scale_factor
-    #    size_diff = tmp_crop_size * (scale_factor - min(scale_factor))
-    #    tmp_5pts = tmp_5pts + size_diff / 2
-    tmp_crop_size = size_bf_outer_pad
-
-    # 3) add outer_padding to make output_size
-    reference_5point = tmp_5pts + np.array(outer_padding)
-    tmp_crop_size = output_size
-
-    return reference_5point
-
-
-def get_affine_transform_matrix(src_pts, dst_pts):
-    """
-    Function:
-    ----------
-        get affine transform matrix 'tfm' from src_pts to dst_pts
-    Parameters:
-    ----------
-        @src_pts: Kx2 np.array
-            source points matrix, each row is a pair of coordinates (x, y)
-        @dst_pts: Kx2 np.array
-            destination points matrix, each row is a pair of coordinates (x, y)
-    Returns:
-    ----------
-        @tfm: 2x3 np.array
-            transform matrix from src_pts to dst_pts
-    """
-
-    tfm = np.float32([[1, 0, 0], [0, 1, 0]])
-    n_pts = src_pts.shape[0]
-    ones = np.ones((n_pts, 1), src_pts.dtype)
-    src_pts_ = np.hstack([src_pts, ones])
-    dst_pts_ = np.hstack([dst_pts, ones])
-
-    A, res, rank, s = np.linalg.lstsq(src_pts_, dst_pts_)
-
-    if rank == 3:
-        tfm = np.float32([[A[0, 0], A[1, 0], A[2, 0]], [A[0, 1], A[1, 1], A[2, 1]]])
-    elif rank == 2:
-        tfm = np.float32([[A[0, 0], A[1, 0], 0], [A[0, 1], A[1, 1], 0]])
-
-    return tfm
-
-
-def warp_and_crop_face(src_img, facial_pts, reference_pts=None, crop_size=(96, 112), align_type='smilarity'):
-    """
-    Function:
-    ----------
-        apply affine transform 'trans' to uv
-    Parameters:
-    ----------
-        @src_img: 3x3 np.array
-            input image
-        @facial_pts: could be
-            1)a list of K coordinates (x,y)
-        or
-            2) Kx2 or 2xK np.array
-            each row or col is a pair of coordinates (x, y)
-        @reference_pts: could be
-            1) a list of K coordinates (x,y)
-        or
-            2) Kx2 or 2xK np.array
-            each row or col is a pair of coordinates (x, y)
-        or
-            3) None
-            if None, use default reference facial points
-        @crop_size: (w, h)
-            output face image size
-        @align_type: transform type, could be one of
-            1) 'similarity': use similarity transform
-            2) 'cv2_affine': use the first 3 points to do affine transform,
-                    by calling cv2.getAffineTransform()
-            3) 'affine': use all points to do affine transform
-    Returns:
-    ----------
-        @face_img: output face image with size (w, h) = @crop_size
-    """
-
-    if reference_pts is None:
-        if crop_size[0] == 96 and crop_size[1] == 112:
-            reference_pts = REFERENCE_FACIAL_POINTS
-        else:
-            default_square = False
-            inner_padding_factor = 0
-            outer_padding = (0, 0)
-            output_size = crop_size
-
-            reference_pts = get_reference_facial_points(output_size, inner_padding_factor, outer_padding,
-                                                        default_square)
-
-    ref_pts = np.float32(reference_pts)
-    ref_pts_shp = ref_pts.shape
-    if max(ref_pts_shp) < 3 or min(ref_pts_shp) != 2:
-        raise FaceWarpException('reference_pts.shape must be (K,2) or (2,K) and K>2')
-
-    if ref_pts_shp[0] == 2:
-        ref_pts = ref_pts.T
-
-    src_pts = np.float32(facial_pts)
-    src_pts_shp = src_pts.shape
-    if max(src_pts_shp) < 3 or min(src_pts_shp) != 2:
-        raise FaceWarpException('facial_pts.shape must be (K,2) or (2,K) and K>2')
-
-    if src_pts_shp[0] == 2:
-        src_pts = src_pts.T
-
-    if src_pts.shape != ref_pts.shape:
-        raise FaceWarpException('facial_pts and reference_pts must have the same shape')
-
-    if align_type == 'cv2_affine':
-        tfm = cv2.getAffineTransform(src_pts[0:3], ref_pts[0:3])
-    elif align_type == 'affine':
-        tfm = get_affine_transform_matrix(src_pts, ref_pts)
-    else:
-        tfm = get_similarity_transform_for_cv2(src_pts, ref_pts)
-
-    face_img = cv2.warpAffine(src_img, tfm, (crop_size[0], crop_size[1]))
-
-    return face_img

extras/facexlib/detection/matlab_cp2tform.py

@@ -1,317 +0,0 @@
-import numpy as np
-from numpy.linalg import inv, lstsq
-from numpy.linalg import matrix_rank as rank
-from numpy.linalg import norm
-
-
-class MatlabCp2tormException(Exception):
-
-    def __str__(self):
-        return 'In File {}:{}'.format(__file__, super.__str__(self))
-
-
-def tformfwd(trans, uv):
-    """
-    Function:
-    ----------
-        apply affine transform 'trans' to uv
-
-    Parameters:
-    ----------
-        @trans: 3x3 np.array
-            transform matrix
-        @uv: Kx2 np.array
-            each row is a pair of coordinates (x, y)
-
-    Returns:
-    ----------
-        @xy: Kx2 np.array
-            each row is a pair of transformed coordinates (x, y)
-    """
-    uv = np.hstack((uv, np.ones((uv.shape[0], 1))))
-    xy = np.dot(uv, trans)
-    xy = xy[:, 0:-1]
-    return xy
-
-
-def tforminv(trans, uv):
-    """
-    Function:
-    ----------
-        apply the inverse of affine transform 'trans' to uv
-
-    Parameters:
-    ----------
-        @trans: 3x3 np.array
-            transform matrix
-        @uv: Kx2 np.array
-            each row is a pair of coordinates (x, y)
-
-    Returns:
-    ----------
-        @xy: Kx2 np.array
-            each row is a pair of inverse-transformed coordinates (x, y)
-    """
-    Tinv = inv(trans)
-    xy = tformfwd(Tinv, uv)
-    return xy
-
-
-def findNonreflectiveSimilarity(uv, xy, options=None):
-    options = {'K': 2}
-
-    K = options['K']
-    M = xy.shape[0]
-    x = xy[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
-    y = xy[:, 1].reshape((-1, 1))  # use reshape to keep a column vector
-
-    tmp1 = np.hstack((x, y, np.ones((M, 1)), np.zeros((M, 1))))
-    tmp2 = np.hstack((y, -x, np.zeros((M, 1)), np.ones((M, 1))))
-    X = np.vstack((tmp1, tmp2))
-
-    u = uv[:, 0].reshape((-1, 1))  # use reshape to keep a column vector
-    v = uv[:, 1].reshape((-1, 1))  # use reshape to keep a column vector
-    U = np.vstack((u, v))
-
-    # We know that X * r = U
-    if rank(X) >= 2 * K:
-        r, _, _, _ = lstsq(X, U, rcond=-1)
-        r = np.squeeze(r)
-    else:
-        raise Exception('cp2tform:twoUniquePointsReq')
-    sc = r[0]
-    ss = r[1]
-    tx = r[2]
-    ty = r[3]
-
-    Tinv = np.array([[sc, -ss, 0], [ss, sc, 0], [tx, ty, 1]])
-    T = inv(Tinv)
-    T[:, 2] = np.array([0, 0, 1])
-
-    return T, Tinv
-
-
-def findSimilarity(uv, xy, options=None):
-    options = {'K': 2}
-
-    #    uv = np.array(uv)
-    #    xy = np.array(xy)
-
-    # Solve for trans1
-    trans1, trans1_inv = findNonreflectiveSimilarity(uv, xy, options)
-
-    # Solve for trans2
-
-    # manually reflect the xy data across the Y-axis
-    xyR = xy
-    xyR[:, 0] = -1 * xyR[:, 0]
-
-    trans2r, trans2r_inv = findNonreflectiveSimilarity(uv, xyR, options)
-
-    # manually reflect the tform to undo the reflection done on xyR
-    TreflectY = np.array([[-1, 0, 0], [0, 1, 0], [0, 0, 1]])
-
-    trans2 = np.dot(trans2r, TreflectY)
-
-    # Figure out if trans1 or trans2 is better
-    xy1 = tformfwd(trans1, uv)
-    norm1 = norm(xy1 - xy)
-
-    xy2 = tformfwd(trans2, uv)
-    norm2 = norm(xy2 - xy)
-
-    if norm1 <= norm2:
-        return trans1, trans1_inv
-    else:
-        trans2_inv = inv(trans2)
-        return trans2, trans2_inv
-
-
-def get_similarity_transform(src_pts, dst_pts, reflective=True):
-    """
-    Function:
-    ----------
-        Find Similarity Transform Matrix 'trans':
-            u = src_pts[:, 0]
-            v = src_pts[:, 1]
-            x = dst_pts[:, 0]
-            y = dst_pts[:, 1]
-            [x, y, 1] = [u, v, 1] * trans
-
-    Parameters:
-    ----------
-        @src_pts: Kx2 np.array
-            source points, each row is a pair of coordinates (x, y)
-        @dst_pts: Kx2 np.array
-            destination points, each row is a pair of transformed
-            coordinates (x, y)
-        @reflective: True or False
-            if True:
-                use reflective similarity transform
-            else:
-                use non-reflective similarity transform
-
-    Returns:
-    ----------
-       @trans: 3x3 np.array
-            transform matrix from uv to xy
-        trans_inv: 3x3 np.array
-            inverse of trans, transform matrix from xy to uv
-    """
-
-    if reflective:
-        trans, trans_inv = findSimilarity(src_pts, dst_pts)
-    else:
-        trans, trans_inv = findNonreflectiveSimilarity(src_pts, dst_pts)
-
-    return trans, trans_inv
-
-
-def cvt_tform_mat_for_cv2(trans):
-    """
-    Function:
-    ----------
-        Convert Transform Matrix 'trans' into 'cv2_trans' which could be
-        directly used by cv2.warpAffine():
-            u = src_pts[:, 0]
-            v = src_pts[:, 1]
-            x = dst_pts[:, 0]
-            y = dst_pts[:, 1]
-            [x, y].T = cv_trans * [u, v, 1].T
-
-    Parameters:
-    ----------
-        @trans: 3x3 np.array
-            transform matrix from uv to xy
-
-    Returns:
-    ----------
-        @cv2_trans: 2x3 np.array
-            transform matrix from src_pts to dst_pts, could be directly used
-            for cv2.warpAffine()
-    """
-    cv2_trans = trans[:, 0:2].T
-
-    return cv2_trans
-
-
-def get_similarity_transform_for_cv2(src_pts, dst_pts, reflective=True):
-    """
-    Function:
-    ----------
-        Find Similarity Transform Matrix 'cv2_trans' which could be
-        directly used by cv2.warpAffine():
-            u = src_pts[:, 0]
-            v = src_pts[:, 1]
-            x = dst_pts[:, 0]
-            y = dst_pts[:, 1]
-            [x, y].T = cv_trans * [u, v, 1].T
-
-    Parameters:
-    ----------
-        @src_pts: Kx2 np.array
-            source points, each row is a pair of coordinates (x, y)
-        @dst_pts: Kx2 np.array
-            destination points, each row is a pair of transformed
-            coordinates (x, y)
-        reflective: True or False
-            if True:
-                use reflective similarity transform
-            else:
-                use non-reflective similarity transform
-
-    Returns:
-    ----------
-        @cv2_trans: 2x3 np.array
-            transform matrix from src_pts to dst_pts, could be directly used
-            for cv2.warpAffine()
-    """
-    trans, trans_inv = get_similarity_transform(src_pts, dst_pts, reflective)
-    cv2_trans = cvt_tform_mat_for_cv2(trans)
-
-    return cv2_trans
-
-
-if __name__ == '__main__':
-    """
-    u = [0, 6, -2]
-    v = [0, 3, 5]
-    x = [-1, 0, 4]
-    y = [-1, -10, 4]
-
-    # In Matlab, run:
-    #
-    #   uv = [u'; v'];
-    #   xy = [x'; y'];
-    #   tform_sim=cp2tform(uv,xy,'similarity');
-    #
-    #   trans = tform_sim.tdata.T
-    #   ans =
-    #       -0.0764   -1.6190         0
-    #        1.6190   -0.0764         0
-    #       -3.2156    0.0290    1.0000
-    #   trans_inv = tform_sim.tdata.Tinv
-    #    ans =
-    #
-    #       -0.0291    0.6163         0
-    #       -0.6163   -0.0291         0
-    #       -0.0756    1.9826    1.0000
-    #    xy_m=tformfwd(tform_sim, u,v)
-    #
-    #    xy_m =
-    #
-    #       -3.2156    0.0290
-    #        1.1833   -9.9143
-    #        5.0323    2.8853
-    #    uv_m=tforminv(tform_sim, x,y)
-    #
-    #    uv_m =
-    #
-    #        0.5698    1.3953
-    #        6.0872    2.2733
-    #       -2.6570    4.3314
-    """
-    u = [0, 6, -2]
-    v = [0, 3, 5]
-    x = [-1, 0, 4]
-    y = [-1, -10, 4]
-
-    uv = np.array((u, v)).T
-    xy = np.array((x, y)).T
-
-    print('\n--->uv:')
-    print(uv)
-    print('\n--->xy:')
-    print(xy)
-
-    trans, trans_inv = get_similarity_transform(uv, xy)
-
-    print('\n--->trans matrix:')
-    print(trans)
-
-    print('\n--->trans_inv matrix:')
-    print(trans_inv)
-
-    print('\n---> apply transform to uv')
-    print('\nxy_m = uv_augmented * trans')
-    uv_aug = np.hstack((uv, np.ones((uv.shape[0], 1))))
-    xy_m = np.dot(uv_aug, trans)
-    print(xy_m)
-
-    print('\nxy_m = tformfwd(trans, uv)')
-    xy_m = tformfwd(trans, uv)
-    print(xy_m)
-
-    print('\n---> apply inverse transform to xy')
-    print('\nuv_m = xy_augmented * trans_inv')
-    xy_aug = np.hstack((xy, np.ones((xy.shape[0], 1))))
-    uv_m = np.dot(xy_aug, trans_inv)
-    print(uv_m)
-
-    print('\nuv_m = tformfwd(trans_inv, xy)')
-    uv_m = tformfwd(trans_inv, xy)
-    print(uv_m)
-
-    uv_m = tforminv(trans, xy)
-    print('\nuv_m = tforminv(trans, xy)')
-    print(uv_m)

extras/facexlib/detection/retinaface.py

@@ -1,366 +0,0 @@
-import cv2
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from PIL import Image
-from torchvision.models._utils import IntermediateLayerGetter as IntermediateLayerGetter
-
-from extras.facexlib.detection.align_trans import get_reference_facial_points, warp_and_crop_face
-from extras.facexlib.detection.retinaface_net import FPN, SSH, MobileNetV1, make_bbox_head, make_class_head, make_landmark_head
-from extras.facexlib.detection.retinaface_utils import (PriorBox, batched_decode, batched_decode_landm, decode, decode_landm,
-                                                 py_cpu_nms)
-
-
-def generate_config(network_name):
-
-    cfg_mnet = {
-        'name': 'mobilenet0.25',
-        'min_sizes': [[16, 32], [64, 128], [256, 512]],
-        'steps': [8, 16, 32],
-        'variance': [0.1, 0.2],
-        'clip': False,
-        'loc_weight': 2.0,
-        'gpu_train': True,
-        'batch_size': 32,
-        'ngpu': 1,
-        'epoch': 250,
-        'decay1': 190,
-        'decay2': 220,
-        'image_size': 640,
-        'return_layers': {
-            'stage1': 1,
-            'stage2': 2,
-            'stage3': 3
-        },
-        'in_channel': 32,
-        'out_channel': 64
-    }
-
-    cfg_re50 = {
-        'name': 'Resnet50',
-        'min_sizes': [[16, 32], [64, 128], [256, 512]],
-        'steps': [8, 16, 32],
-        'variance': [0.1, 0.2],
-        'clip': False,
-        'loc_weight': 2.0,
-        'gpu_train': True,
-        'batch_size': 24,
-        'ngpu': 4,
-        'epoch': 100,
-        'decay1': 70,
-        'decay2': 90,
-        'image_size': 840,
-        'return_layers': {
-            'layer2': 1,
-            'layer3': 2,
-            'layer4': 3
-        },
-        'in_channel': 256,
-        'out_channel': 256
-    }
-
-    if network_name == 'mobile0.25':
-        return cfg_mnet
-    elif network_name == 'resnet50':
-        return cfg_re50
-    else:
-        raise NotImplementedError(f'network_name={network_name}')
-
-
-class RetinaFace(nn.Module):
-
-    def __init__(self, network_name='resnet50', half=False, phase='test', device=None):
-        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device is None else device
-
-        super(RetinaFace, self).__init__()
-        self.half_inference = half
-        cfg = generate_config(network_name)
-        self.backbone = cfg['name']
-
-        self.model_name = f'retinaface_{network_name}'
-        self.cfg = cfg
-        self.phase = phase
-        self.target_size, self.max_size = 1600, 2150
-        self.resize, self.scale, self.scale1 = 1., None, None
-        self.mean_tensor = torch.tensor([[[[104.]], [[117.]], [[123.]]]], device=self.device)
-        self.reference = get_reference_facial_points(default_square=True)
-        # Build network.
-        backbone = None
-        if cfg['name'] == 'mobilenet0.25':
-            backbone = MobileNetV1()
-            self.body = IntermediateLayerGetter(backbone, cfg['return_layers'])
-        elif cfg['name'] == 'Resnet50':
-            import torchvision.models as models
-            backbone = models.resnet50(weights=None)
-            self.body = IntermediateLayerGetter(backbone, cfg['return_layers'])
-
-        in_channels_stage2 = cfg['in_channel']
-        in_channels_list = [
-            in_channels_stage2 * 2,
-            in_channels_stage2 * 4,
-            in_channels_stage2 * 8,
-        ]
-
-        out_channels = cfg['out_channel']
-        self.fpn = FPN(in_channels_list, out_channels)
-        self.ssh1 = SSH(out_channels, out_channels)
-        self.ssh2 = SSH(out_channels, out_channels)
-        self.ssh3 = SSH(out_channels, out_channels)
-
-        self.ClassHead = make_class_head(fpn_num=3, inchannels=cfg['out_channel'])
-        self.BboxHead = make_bbox_head(fpn_num=3, inchannels=cfg['out_channel'])
-        self.LandmarkHead = make_landmark_head(fpn_num=3, inchannels=cfg['out_channel'])
-
-        self.to(self.device)
-        self.eval()
-        if self.half_inference:
-            self.half()
-
-    def forward(self, inputs):
-        out = self.body(inputs)
-
-        if self.backbone == 'mobilenet0.25' or self.backbone == 'Resnet50':
-            out = list(out.values())
-        # FPN
-        fpn = self.fpn(out)
-
-        # SSH
-        feature1 = self.ssh1(fpn[0])
-        feature2 = self.ssh2(fpn[1])
-        feature3 = self.ssh3(fpn[2])
-        features = [feature1, feature2, feature3]
-
-        bbox_regressions = torch.cat([self.BboxHead[i](feature) for i, feature in enumerate(features)], dim=1)
-        classifications = torch.cat([self.ClassHead[i](feature) for i, feature in enumerate(features)], dim=1)
-        tmp = [self.LandmarkHead[i](feature) for i, feature in enumerate(features)]
-        ldm_regressions = (torch.cat(tmp, dim=1))
-
-        if self.phase == 'train':
-            output = (bbox_regressions, classifications, ldm_regressions)
-        else:
-            output = (bbox_regressions, F.softmax(classifications, dim=-1), ldm_regressions)
-        return output
-
-    def __detect_faces(self, inputs):
-        # get scale
-        height, width = inputs.shape[2:]
-        self.scale = torch.tensor([width, height, width, height], dtype=torch.float32, device=self.device)
-        tmp = [width, height, width, height, width, height, width, height, width, height]
-        self.scale1 = torch.tensor(tmp, dtype=torch.float32, device=self.device)
-
-        # forawrd
-        inputs = inputs.to(self.device)
-        if self.half_inference:
-            inputs = inputs.half()
-        loc, conf, landmarks = self(inputs)
-
-        # get priorbox
-        priorbox = PriorBox(self.cfg, image_size=inputs.shape[2:])
-        priors = priorbox.forward().to(self.device)
-
-        return loc, conf, landmarks, priors
-
-    # single image detection
-    def transform(self, image, use_origin_size):
-        # convert to opencv format
-        if isinstance(image, Image.Image):
-            image = cv2.cvtColor(np.asarray(image), cv2.COLOR_RGB2BGR)
-        image = image.astype(np.float32)
-
-        # testing scale
-        im_size_min = np.min(image.shape[0:2])
-        im_size_max = np.max(image.shape[0:2])
-        resize = float(self.target_size) / float(im_size_min)
-
-        # prevent bigger axis from being more than max_size
-        if np.round(resize * im_size_max) > self.max_size:
-            resize = float(self.max_size) / float(im_size_max)
-        resize = 1 if use_origin_size else resize
-
-        # resize
-        if resize != 1:
-            image = cv2.resize(image, None, None, fx=resize, fy=resize, interpolation=cv2.INTER_LINEAR)
-
-        # convert to torch.tensor format
-        # image -= (104, 117, 123)
-        image = image.transpose(2, 0, 1)
-        image = torch.from_numpy(image).unsqueeze(0)
-
-        return image, resize
-
-    def detect_faces(
-        self,
-        image,
-        conf_threshold=0.8,
-        nms_threshold=0.4,
-        use_origin_size=True,
-    ):
-        image, self.resize = self.transform(image, use_origin_size)
-        image = image.to(self.device)
-        if self.half_inference:
-            image = image.half()
-        image = image - self.mean_tensor
-
-        loc, conf, landmarks, priors = self.__detect_faces(image)
-
-        boxes = decode(loc.data.squeeze(0), priors.data, self.cfg['variance'])
-        boxes = boxes * self.scale / self.resize
-        boxes = boxes.cpu().numpy()
-
-        scores = conf.squeeze(0).data.cpu().numpy()[:, 1]
-
-        landmarks = decode_landm(landmarks.squeeze(0), priors, self.cfg['variance'])
-        landmarks = landmarks * self.scale1 / self.resize
-        landmarks = landmarks.cpu().numpy()
-
-        # ignore low scores
-        inds = np.where(scores > conf_threshold)[0]
-        boxes, landmarks, scores = boxes[inds], landmarks[inds], scores[inds]
-
-        # sort
-        order = scores.argsort()[::-1]
-        boxes, landmarks, scores = boxes[order], landmarks[order], scores[order]
-
-        # do NMS
-        bounding_boxes = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)
-        keep = py_cpu_nms(bounding_boxes, nms_threshold)
-        bounding_boxes, landmarks = bounding_boxes[keep, :], landmarks[keep]
-        # self.t['forward_pass'].toc()
-        # print(self.t['forward_pass'].average_time)
-        # import sys
-        # sys.stdout.flush()
-        return np.concatenate((bounding_boxes, landmarks), axis=1)
-
-    def __align_multi(self, image, boxes, landmarks, limit=None):
-
-        if len(boxes) < 1:
-            return [], []
-
-        if limit:
-            boxes = boxes[:limit]
-            landmarks = landmarks[:limit]
-
-        faces = []
-        for landmark in landmarks:
-            facial5points = [[landmark[2 * j], landmark[2 * j + 1]] for j in range(5)]
-
-            warped_face = warp_and_crop_face(np.array(image), facial5points, self.reference, crop_size=(112, 112))
-            faces.append(warped_face)
-
-        return np.concatenate((boxes, landmarks), axis=1), faces
-
-    def align_multi(self, img, conf_threshold=0.8, limit=None):
-
-        rlt = self.detect_faces(img, conf_threshold=conf_threshold)
-        boxes, landmarks = rlt[:, 0:5], rlt[:, 5:]
-
-        return self.__align_multi(img, boxes, landmarks, limit)
-
-    # batched detection
-    def batched_transform(self, frames, use_origin_size):
-        """
-        Arguments:
-            frames: a list of PIL.Image, or torch.Tensor(shape=[n, h, w, c],
-                type=np.float32, BGR format).
-            use_origin_size: whether to use origin size.
-        """
-        from_PIL = True if isinstance(frames[0], Image.Image) else False
-
-        # convert to opencv format
-        if from_PIL:
-            frames = [cv2.cvtColor(np.asarray(frame), cv2.COLOR_RGB2BGR) for frame in frames]
-            frames = np.asarray(frames, dtype=np.float32)
-
-        # testing scale
-        im_size_min = np.min(frames[0].shape[0:2])
-        im_size_max = np.max(frames[0].shape[0:2])
-        resize = float(self.target_size) / float(im_size_min)
-
-        # prevent bigger axis from being more than max_size
-        if np.round(resize * im_size_max) > self.max_size:
-            resize = float(self.max_size) / float(im_size_max)
-        resize = 1 if use_origin_size else resize
-
-        # resize
-        if resize != 1:
-            if not from_PIL:
-                frames = F.interpolate(frames, scale_factor=resize)
-            else:
-                frames = [
-                    cv2.resize(frame, None, None, fx=resize, fy=resize, interpolation=cv2.INTER_LINEAR)
-                    for frame in frames
-                ]
-
-        # convert to torch.tensor format
-        if not from_PIL:
-            frames = frames.transpose(1, 2).transpose(1, 3).contiguous()
-        else:
-            frames = frames.transpose((0, 3, 1, 2))
-            frames = torch.from_numpy(frames)
-
-        return frames, resize
-
-    def batched_detect_faces(self, frames, conf_threshold=0.8, nms_threshold=0.4, use_origin_size=True):
-        """
-        Arguments:
-            frames: a list of PIL.Image, or np.array(shape=[n, h, w, c],
-                type=np.uint8, BGR format).
-            conf_threshold: confidence threshold.
-            nms_threshold: nms threshold.
-            use_origin_size: whether to use origin size.
-        Returns:
-            final_bounding_boxes: list of np.array ([n_boxes, 5],
-                type=np.float32).
-            final_landmarks: list of np.array ([n_boxes, 10], type=np.float32).
-        """
-        # self.t['forward_pass'].tic()
-        frames, self.resize = self.batched_transform(frames, use_origin_size)
-        frames = frames.to(self.device)
-        frames = frames - self.mean_tensor
-
-        b_loc, b_conf, b_landmarks, priors = self.__detect_faces(frames)
-
-        final_bounding_boxes, final_landmarks = [], []
-
-        # decode
-        priors = priors.unsqueeze(0)
-        b_loc = batched_decode(b_loc, priors, self.cfg['variance']) * self.scale / self.resize
-        b_landmarks = batched_decode_landm(b_landmarks, priors, self.cfg['variance']) * self.scale1 / self.resize
-        b_conf = b_conf[:, :, 1]
-
-        # index for selection
-        b_indice = b_conf > conf_threshold
-
-        # concat
-        b_loc_and_conf = torch.cat((b_loc, b_conf.unsqueeze(-1)), dim=2).float()
-
-        for pred, landm, inds in zip(b_loc_and_conf, b_landmarks, b_indice):
-
-            # ignore low scores
-            pred, landm = pred[inds, :], landm[inds, :]
-            if pred.shape[0] == 0:
-                final_bounding_boxes.append(np.array([], dtype=np.float32))
-                final_landmarks.append(np.array([], dtype=np.float32))
-                continue
-
-            # sort
-            # order = score.argsort(descending=True)
-            # box, landm, score = box[order], landm[order], score[order]
-
-            # to CPU
-            bounding_boxes, landm = pred.cpu().numpy(), landm.cpu().numpy()
-
-            # NMS
-            keep = py_cpu_nms(bounding_boxes, nms_threshold)
-            bounding_boxes, landmarks = bounding_boxes[keep, :], landm[keep]
-
-            # append
-            final_bounding_boxes.append(bounding_boxes)
-            final_landmarks.append(landmarks)
-        # self.t['forward_pass'].toc(average=True)
-        # self.batch_time += self.t['forward_pass'].diff
-        # self.total_frame += len(frames)
-        # print(self.batch_time / self.total_frame)
-
-        return final_bounding_boxes, final_landmarks

extras/facexlib/detection/retinaface_net.py

@@ -1,196 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-def conv_bn(inp, oup, stride=1, leaky=0):
-    return nn.Sequential(
-        nn.Conv2d(inp, oup, 3, stride, 1, bias=False), nn.BatchNorm2d(oup),
-        nn.LeakyReLU(negative_slope=leaky, inplace=True))
-
-
-def conv_bn_no_relu(inp, oup, stride):
-    return nn.Sequential(
-        nn.Conv2d(inp, oup, 3, stride, 1, bias=False),
-        nn.BatchNorm2d(oup),
-    )
-
-
-def conv_bn1X1(inp, oup, stride, leaky=0):
-    return nn.Sequential(
-        nn.Conv2d(inp, oup, 1, stride, padding=0, bias=False), nn.BatchNorm2d(oup),
-        nn.LeakyReLU(negative_slope=leaky, inplace=True))
-
-
-def conv_dw(inp, oup, stride, leaky=0.1):
-    return nn.Sequential(
-        nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False),
-        nn.BatchNorm2d(inp),
-        nn.LeakyReLU(negative_slope=leaky, inplace=True),
-        nn.Conv2d(inp, oup, 1, 1, 0, bias=False),
-        nn.BatchNorm2d(oup),
-        nn.LeakyReLU(negative_slope=leaky, inplace=True),
-    )
-
-
-class SSH(nn.Module):
-
-    def __init__(self, in_channel, out_channel):
-        super(SSH, self).__init__()
-        assert out_channel % 4 == 0
-        leaky = 0
-        if (out_channel <= 64):
-            leaky = 0.1
-        self.conv3X3 = conv_bn_no_relu(in_channel, out_channel // 2, stride=1)
-
-        self.conv5X5_1 = conv_bn(in_channel, out_channel // 4, stride=1, leaky=leaky)
-        self.conv5X5_2 = conv_bn_no_relu(out_channel // 4, out_channel // 4, stride=1)
-
-        self.conv7X7_2 = conv_bn(out_channel // 4, out_channel // 4, stride=1, leaky=leaky)
-        self.conv7x7_3 = conv_bn_no_relu(out_channel // 4, out_channel // 4, stride=1)
-
-    def forward(self, input):
-        conv3X3 = self.conv3X3(input)
-
-        conv5X5_1 = self.conv5X5_1(input)
-        conv5X5 = self.conv5X5_2(conv5X5_1)
-
-        conv7X7_2 = self.conv7X7_2(conv5X5_1)
-        conv7X7 = self.conv7x7_3(conv7X7_2)
-
-        out = torch.cat([conv3X3, conv5X5, conv7X7], dim=1)
-        out = F.relu(out)
-        return out
-
-
-class FPN(nn.Module):
-
-    def __init__(self, in_channels_list, out_channels):
-        super(FPN, self).__init__()
-        leaky = 0
-        if (out_channels <= 64):
-            leaky = 0.1
-        self.output1 = conv_bn1X1(in_channels_list[0], out_channels, stride=1, leaky=leaky)
-        self.output2 = conv_bn1X1(in_channels_list[1], out_channels, stride=1, leaky=leaky)
-        self.output3 = conv_bn1X1(in_channels_list[2], out_channels, stride=1, leaky=leaky)
-
-        self.merge1 = conv_bn(out_channels, out_channels, leaky=leaky)
-        self.merge2 = conv_bn(out_channels, out_channels, leaky=leaky)
-
-    def forward(self, input):
-        # names = list(input.keys())
-        # input = list(input.values())
-
-        output1 = self.output1(input[0])
-        output2 = self.output2(input[1])
-        output3 = self.output3(input[2])
-
-        up3 = F.interpolate(output3, size=[output2.size(2), output2.size(3)], mode='nearest')
-        output2 = output2 + up3
-        output2 = self.merge2(output2)
-
-        up2 = F.interpolate(output2, size=[output1.size(2), output1.size(3)], mode='nearest')
-        output1 = output1 + up2
-        output1 = self.merge1(output1)
-
-        out = [output1, output2, output3]
-        return out
-
-
-class MobileNetV1(nn.Module):
-
-    def __init__(self):
-        super(MobileNetV1, self).__init__()
-        self.stage1 = nn.Sequential(
-            conv_bn(3, 8, 2, leaky=0.1),  # 3
-            conv_dw(8, 16, 1),  # 7
-            conv_dw(16, 32, 2),  # 11
-            conv_dw(32, 32, 1),  # 19
-            conv_dw(32, 64, 2),  # 27
-            conv_dw(64, 64, 1),  # 43
-        )
-        self.stage2 = nn.Sequential(
-            conv_dw(64, 128, 2),  # 43 + 16 = 59
-            conv_dw(128, 128, 1),  # 59 + 32 = 91
-            conv_dw(128, 128, 1),  # 91 + 32 = 123
-            conv_dw(128, 128, 1),  # 123 + 32 = 155
-            conv_dw(128, 128, 1),  # 155 + 32 = 187
-            conv_dw(128, 128, 1),  # 187 + 32 = 219
-        )
-        self.stage3 = nn.Sequential(
-            conv_dw(128, 256, 2),  # 219 +3 2 = 241
-            conv_dw(256, 256, 1),  # 241 + 64 = 301
-        )
-        self.avg = nn.AdaptiveAvgPool2d((1, 1))
-        self.fc = nn.Linear(256, 1000)
-
-    def forward(self, x):
-        x = self.stage1(x)
-        x = self.stage2(x)
-        x = self.stage3(x)
-        x = self.avg(x)
-        # x = self.model(x)
-        x = x.view(-1, 256)
-        x = self.fc(x)
-        return x
-
-
-class ClassHead(nn.Module):
-
-    def __init__(self, inchannels=512, num_anchors=3):
-        super(ClassHead, self).__init__()
-        self.num_anchors = num_anchors
-        self.conv1x1 = nn.Conv2d(inchannels, self.num_anchors * 2, kernel_size=(1, 1), stride=1, padding=0)
-
-    def forward(self, x):
-        out = self.conv1x1(x)
-        out = out.permute(0, 2, 3, 1).contiguous()
-
-        return out.view(out.shape[0], -1, 2)
-
-
-class BboxHead(nn.Module):
-
-    def __init__(self, inchannels=512, num_anchors=3):
-        super(BboxHead, self).__init__()
-        self.conv1x1 = nn.Conv2d(inchannels, num_anchors * 4, kernel_size=(1, 1), stride=1, padding=0)
-
-    def forward(self, x):
-        out = self.conv1x1(x)
-        out = out.permute(0, 2, 3, 1).contiguous()
-
-        return out.view(out.shape[0], -1, 4)
-
-
-class LandmarkHead(nn.Module):
-
-    def __init__(self, inchannels=512, num_anchors=3):
-        super(LandmarkHead, self).__init__()
-        self.conv1x1 = nn.Conv2d(inchannels, num_anchors * 10, kernel_size=(1, 1), stride=1, padding=0)
-
-    def forward(self, x):
-        out = self.conv1x1(x)
-        out = out.permute(0, 2, 3, 1).contiguous()
-
-        return out.view(out.shape[0], -1, 10)
-
-
-def make_class_head(fpn_num=3, inchannels=64, anchor_num=2):
-    classhead = nn.ModuleList()
-    for i in range(fpn_num):
-        classhead.append(ClassHead(inchannels, anchor_num))
-    return classhead
-
-
-def make_bbox_head(fpn_num=3, inchannels=64, anchor_num=2):
-    bboxhead = nn.ModuleList()
-    for i in range(fpn_num):
-        bboxhead.append(BboxHead(inchannels, anchor_num))
-    return bboxhead
-
-
-def make_landmark_head(fpn_num=3, inchannels=64, anchor_num=2):
-    landmarkhead = nn.ModuleList()
-    for i in range(fpn_num):
-        landmarkhead.append(LandmarkHead(inchannels, anchor_num))
-    return landmarkhead

extras/facexlib/detection/retinaface_utils.py

@@ -1,421 +0,0 @@
-import numpy as np
-import torch
-import torchvision
-from itertools import product as product
-from math import ceil
-
-
-class PriorBox(object):
-
-    def __init__(self, cfg, image_size=None, phase='train'):
-        super(PriorBox, self).__init__()
-        self.min_sizes = cfg['min_sizes']
-        self.steps = cfg['steps']
-        self.clip = cfg['clip']
-        self.image_size = image_size
-        self.feature_maps = [[ceil(self.image_size[0] / step), ceil(self.image_size[1] / step)] for step in self.steps]
-        self.name = 's'
-
-    def forward(self):
-        anchors = []
-        for k, f in enumerate(self.feature_maps):
-            min_sizes = self.min_sizes[k]
-            for i, j in product(range(f[0]), range(f[1])):
-                for min_size in min_sizes:
-                    s_kx = min_size / self.image_size[1]
-                    s_ky = min_size / self.image_size[0]
-                    dense_cx = [x * self.steps[k] / self.image_size[1] for x in [j + 0.5]]
-                    dense_cy = [y * self.steps[k] / self.image_size[0] for y in [i + 0.5]]
-                    for cy, cx in product(dense_cy, dense_cx):
-                        anchors += [cx, cy, s_kx, s_ky]
-
-        # back to torch land
-        output = torch.Tensor(anchors).view(-1, 4)
-        if self.clip:
-            output.clamp_(max=1, min=0)
-        return output
-
-
-def py_cpu_nms(dets, thresh):
-    """Pure Python NMS baseline."""
-    keep = torchvision.ops.nms(
-        boxes=torch.Tensor(dets[:, :4]),
-        scores=torch.Tensor(dets[:, 4]),
-        iou_threshold=thresh,
-    )
-
-    return list(keep)
-
-
-def point_form(boxes):
-    """ Convert prior_boxes to (xmin, ymin, xmax, ymax)
-    representation for comparison to point form ground truth data.
-    Args:
-        boxes: (tensor) center-size default boxes from priorbox layers.
-    Return:
-        boxes: (tensor) Converted xmin, ymin, xmax, ymax form of boxes.
-    """
-    return torch.cat(
-        (
-            boxes[:, :2] - boxes[:, 2:] / 2,  # xmin, ymin
-            boxes[:, :2] + boxes[:, 2:] / 2),
-        1)  # xmax, ymax
-
-
-def center_size(boxes):
-    """ Convert prior_boxes to (cx, cy, w, h)
-    representation for comparison to center-size form ground truth data.
-    Args:
-        boxes: (tensor) point_form boxes
-    Return:
-        boxes: (tensor) Converted xmin, ymin, xmax, ymax form of boxes.
-    """
-    return torch.cat(
-        (boxes[:, 2:] + boxes[:, :2]) / 2,  # cx, cy
-        boxes[:, 2:] - boxes[:, :2],
-        1)  # w, h
-
-
-def intersect(box_a, box_b):
-    """ We resize both tensors to [A,B,2] without new malloc:
-    [A,2] -> [A,1,2] -> [A,B,2]
-    [B,2] -> [1,B,2] -> [A,B,2]
-    Then we compute the area of intersect between box_a and box_b.
-    Args:
-      box_a: (tensor) bounding boxes, Shape: [A,4].
-      box_b: (tensor) bounding boxes, Shape: [B,4].
-    Return:
-      (tensor) intersection area, Shape: [A,B].
-    """
-    A = box_a.size(0)
-    B = box_b.size(0)
-    max_xy = torch.min(box_a[:, 2:].unsqueeze(1).expand(A, B, 2), box_b[:, 2:].unsqueeze(0).expand(A, B, 2))
-    min_xy = torch.max(box_a[:, :2].unsqueeze(1).expand(A, B, 2), box_b[:, :2].unsqueeze(0).expand(A, B, 2))
-    inter = torch.clamp((max_xy - min_xy), min=0)
-    return inter[:, :, 0] * inter[:, :, 1]
-
-
-def jaccard(box_a, box_b):
-    """Compute the jaccard overlap of two sets of boxes.  The jaccard overlap
-    is simply the intersection over union of two boxes.  Here we operate on
-    ground truth boxes and default boxes.
-    E.g.:
-        A ∩ B / A ∪ B = A ∩ B / (area(A) + area(B) - A ∩ B)
-    Args:
-        box_a: (tensor) Ground truth bounding boxes, Shape: [num_objects,4]
-        box_b: (tensor) Prior boxes from priorbox layers, Shape: [num_priors,4]
-    Return:
-        jaccard overlap: (tensor) Shape: [box_a.size(0), box_b.size(0)]
-    """
-    inter = intersect(box_a, box_b)
-    area_a = ((box_a[:, 2] - box_a[:, 0]) * (box_a[:, 3] - box_a[:, 1])).unsqueeze(1).expand_as(inter)  # [A,B]
-    area_b = ((box_b[:, 2] - box_b[:, 0]) * (box_b[:, 3] - box_b[:, 1])).unsqueeze(0).expand_as(inter)  # [A,B]
-    union = area_a + area_b - inter
-    return inter / union  # [A,B]
-
-
-def matrix_iou(a, b):
-    """
-    return iou of a and b, numpy version for data augenmentation
-    """
-    lt = np.maximum(a[:, np.newaxis, :2], b[:, :2])
-    rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:])
-
-    area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2)
-    area_a = np.prod(a[:, 2:] - a[:, :2], axis=1)
-    area_b = np.prod(b[:, 2:] - b[:, :2], axis=1)
-    return area_i / (area_a[:, np.newaxis] + area_b - area_i)
-
-
-def matrix_iof(a, b):
-    """
-    return iof of a and b, numpy version for data augenmentation
-    """
-    lt = np.maximum(a[:, np.newaxis, :2], b[:, :2])
-    rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:])
-
-    area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2)
-    area_a = np.prod(a[:, 2:] - a[:, :2], axis=1)
-    return area_i / np.maximum(area_a[:, np.newaxis], 1)
-
-
-def match(threshold, truths, priors, variances, labels, landms, loc_t, conf_t, landm_t, idx):
-    """Match each prior box with the ground truth box of the highest jaccard
-    overlap, encode the bounding boxes, then return the matched indices
-    corresponding to both confidence and location preds.
-    Args:
-        threshold: (float) The overlap threshold used when matching boxes.
-        truths: (tensor) Ground truth boxes, Shape: [num_obj, 4].
-        priors: (tensor) Prior boxes from priorbox layers, Shape: [n_priors,4].
-        variances: (tensor) Variances corresponding to each prior coord,
-            Shape: [num_priors, 4].
-        labels: (tensor) All the class labels for the image, Shape: [num_obj].
-        landms: (tensor) Ground truth landms, Shape [num_obj, 10].
-        loc_t: (tensor) Tensor to be filled w/ encoded location targets.
-        conf_t: (tensor) Tensor to be filled w/ matched indices for conf preds.
-        landm_t: (tensor) Tensor to be filled w/ encoded landm targets.
-        idx: (int) current batch index
-    Return:
-        The matched indices corresponding to 1)location 2)confidence
-        3)landm preds.
-    """
-    # jaccard index
-    overlaps = jaccard(truths, point_form(priors))
-    # (Bipartite Matching)
-    # [1,num_objects] best prior for each ground truth
-    best_prior_overlap, best_prior_idx = overlaps.max(1, keepdim=True)
-
-    # ignore hard gt
-    valid_gt_idx = best_prior_overlap[:, 0] >= 0.2
-    best_prior_idx_filter = best_prior_idx[valid_gt_idx, :]
-    if best_prior_idx_filter.shape[0] <= 0:
-        loc_t[idx] = 0
-        conf_t[idx] = 0
-        return
-
-    # [1,num_priors] best ground truth for each prior
-    best_truth_overlap, best_truth_idx = overlaps.max(0, keepdim=True)
-    best_truth_idx.squeeze_(0)
-    best_truth_overlap.squeeze_(0)
-    best_prior_idx.squeeze_(1)
-    best_prior_idx_filter.squeeze_(1)
-    best_prior_overlap.squeeze_(1)
-    best_truth_overlap.index_fill_(0, best_prior_idx_filter, 2)  # ensure best prior
-    # TODO refactor: index  best_prior_idx with long tensor
-    # ensure every gt matches with its prior of max overlap
-    for j in range(best_prior_idx.size(0)):  # 判别此anchor是预测哪一个boxes
-        best_truth_idx[best_prior_idx[j]] = j
-    matches = truths[best_truth_idx]  # Shape: [num_priors,4] 此处为每一个anchor对应的bbox取出来
-    conf = labels[best_truth_idx]  # Shape: [num_priors]      此处为每一个anchor对应的label取出来
-    conf[best_truth_overlap < threshold] = 0  # label as background   overlap<0.35的全部作为负样本
-    loc = encode(matches, priors, variances)
-
-    matches_landm = landms[best_truth_idx]
-    landm = encode_landm(matches_landm, priors, variances)
-    loc_t[idx] = loc  # [num_priors,4] encoded offsets to learn
-    conf_t[idx] = conf  # [num_priors] top class label for each prior
-    landm_t[idx] = landm
-
-
-def encode(matched, priors, variances):
-    """Encode the variances from the priorbox layers into the ground truth boxes
-    we have matched (based on jaccard overlap) with the prior boxes.
-    Args:
-        matched: (tensor) Coords of ground truth for each prior in point-form
-            Shape: [num_priors, 4].
-        priors: (tensor) Prior boxes in center-offset form
-            Shape: [num_priors,4].
-        variances: (list[float]) Variances of priorboxes
-    Return:
-        encoded boxes (tensor), Shape: [num_priors, 4]
-    """
-
-    # dist b/t match center and prior's center
-    g_cxcy = (matched[:, :2] + matched[:, 2:]) / 2 - priors[:, :2]
-    # encode variance
-    g_cxcy /= (variances[0] * priors[:, 2:])
-    # match wh / prior wh
-    g_wh = (matched[:, 2:] - matched[:, :2]) / priors[:, 2:]
-    g_wh = torch.log(g_wh) / variances[1]
-    # return target for smooth_l1_loss
-    return torch.cat([g_cxcy, g_wh], 1)  # [num_priors,4]
-
-
-def encode_landm(matched, priors, variances):
-    """Encode the variances from the priorbox layers into the ground truth boxes
-    we have matched (based on jaccard overlap) with the prior boxes.
-    Args:
-        matched: (tensor) Coords of ground truth for each prior in point-form
-            Shape: [num_priors, 10].
-        priors: (tensor) Prior boxes in center-offset form
-            Shape: [num_priors,4].
-        variances: (list[float]) Variances of priorboxes
-    Return:
-        encoded landm (tensor), Shape: [num_priors, 10]
-    """
-
-    # dist b/t match center and prior's center
-    matched = torch.reshape(matched, (matched.size(0), 5, 2))
-    priors_cx = priors[:, 0].unsqueeze(1).expand(matched.size(0), 5).unsqueeze(2)
-    priors_cy = priors[:, 1].unsqueeze(1).expand(matched.size(0), 5).unsqueeze(2)
-    priors_w = priors[:, 2].unsqueeze(1).expand(matched.size(0), 5).unsqueeze(2)
-    priors_h = priors[:, 3].unsqueeze(1).expand(matched.size(0), 5).unsqueeze(2)
-    priors = torch.cat([priors_cx, priors_cy, priors_w, priors_h], dim=2)
-    g_cxcy = matched[:, :, :2] - priors[:, :, :2]
-    # encode variance
-    g_cxcy /= (variances[0] * priors[:, :, 2:])
-    # g_cxcy /= priors[:, :, 2:]
-    g_cxcy = g_cxcy.reshape(g_cxcy.size(0), -1)
-    # return target for smooth_l1_loss
-    return g_cxcy
-
-
-# Adapted from https://github.com/Hakuyume/chainer-ssd
-def decode(loc, priors, variances):
-    """Decode locations from predictions using priors to undo
-    the encoding we did for offset regression at train time.
-    Args:
-        loc (tensor): location predictions for loc layers,
-            Shape: [num_priors,4]
-        priors (tensor): Prior boxes in center-offset form.
-            Shape: [num_priors,4].
-        variances: (list[float]) Variances of priorboxes
-    Return:
-        decoded bounding box predictions
-    """
-
-    boxes = torch.cat((priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:],
-                       priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1])), 1)
-    boxes[:, :2] -= boxes[:, 2:] / 2
-    boxes[:, 2:] += boxes[:, :2]
-    return boxes
-
-
-def decode_landm(pre, priors, variances):
-    """Decode landm from predictions using priors to undo
-    the encoding we did for offset regression at train time.
-    Args:
-        pre (tensor): landm predictions for loc layers,
-            Shape: [num_priors,10]
-        priors (tensor): Prior boxes in center-offset form.
-            Shape: [num_priors,4].
-        variances: (list[float]) Variances of priorboxes
-    Return:
-        decoded landm predictions
-    """
-    tmp = (
-        priors[:, :2] + pre[:, :2] * variances[0] * priors[:, 2:],
-        priors[:, :2] + pre[:, 2:4] * variances[0] * priors[:, 2:],
-        priors[:, :2] + pre[:, 4:6] * variances[0] * priors[:, 2:],
-        priors[:, :2] + pre[:, 6:8] * variances[0] * priors[:, 2:],
-        priors[:, :2] + pre[:, 8:10] * variances[0] * priors[:, 2:],
-    )
-    landms = torch.cat(tmp, dim=1)
-    return landms
-
-
-def batched_decode(b_loc, priors, variances):
-    """Decode locations from predictions using priors to undo
-    the encoding we did for offset regression at train time.
-    Args:
-        b_loc (tensor): location predictions for loc layers,
-            Shape: [num_batches,num_priors,4]
-        priors (tensor): Prior boxes in center-offset form.
-            Shape: [1,num_priors,4].
-        variances: (list[float]) Variances of priorboxes
-    Return:
-        decoded bounding box predictions
-    """
-    boxes = (
-        priors[:, :, :2] + b_loc[:, :, :2] * variances[0] * priors[:, :, 2:],
-        priors[:, :, 2:] * torch.exp(b_loc[:, :, 2:] * variances[1]),
-    )
-    boxes = torch.cat(boxes, dim=2)
-
-    boxes[:, :, :2] -= boxes[:, :, 2:] / 2
-    boxes[:, :, 2:] += boxes[:, :, :2]
-    return boxes
-
-
-def batched_decode_landm(pre, priors, variances):
-    """Decode landm from predictions using priors to undo
-    the encoding we did for offset regression at train time.
-    Args:
-        pre (tensor): landm predictions for loc layers,
-            Shape: [num_batches,num_priors,10]
-        priors (tensor): Prior boxes in center-offset form.
-            Shape: [1,num_priors,4].
-        variances: (list[float]) Variances of priorboxes
-    Return:
-        decoded landm predictions
-    """
-    landms = (
-        priors[:, :, :2] + pre[:, :, :2] * variances[0] * priors[:, :, 2:],
-        priors[:, :, :2] + pre[:, :, 2:4] * variances[0] * priors[:, :, 2:],
-        priors[:, :, :2] + pre[:, :, 4:6] * variances[0] * priors[:, :, 2:],
-        priors[:, :, :2] + pre[:, :, 6:8] * variances[0] * priors[:, :, 2:],
-        priors[:, :, :2] + pre[:, :, 8:10] * variances[0] * priors[:, :, 2:],
-    )
-    landms = torch.cat(landms, dim=2)
-    return landms
-
-
-def log_sum_exp(x):
-    """Utility function for computing log_sum_exp while determining
-    This will be used to determine unaveraged confidence loss across
-    all examples in a batch.
-    Args:
-        x (Variable(tensor)): conf_preds from conf layers
-    """
-    x_max = x.data.max()
-    return torch.log(torch.sum(torch.exp(x - x_max), 1, keepdim=True)) + x_max
-
-
-# Original author: Francisco Massa:
-# https://github.com/fmassa/object-detection.torch
-# Ported to PyTorch by Max deGroot (02/01/2017)
-def nms(boxes, scores, overlap=0.5, top_k=200):
-    """Apply non-maximum suppression at test time to avoid detecting too many
-    overlapping bounding boxes for a given object.
-    Args:
-        boxes: (tensor) The location preds for the img, Shape: [num_priors,4].
-        scores: (tensor) The class predscores for the img, Shape:[num_priors].
-        overlap: (float) The overlap thresh for suppressing unnecessary boxes.
-        top_k: (int) The Maximum number of box preds to consider.
-    Return:
-        The indices of the kept boxes with respect to num_priors.
-    """
-
-    keep = torch.Tensor(scores.size(0)).fill_(0).long()
-    if boxes.numel() == 0:
-        return keep
-    x1 = boxes[:, 0]
-    y1 = boxes[:, 1]
-    x2 = boxes[:, 2]
-    y2 = boxes[:, 3]
-    area = torch.mul(x2 - x1, y2 - y1)
-    v, idx = scores.sort(0)  # sort in ascending order
-    # I = I[v >= 0.01]
-    idx = idx[-top_k:]  # indices of the top-k largest vals
-    xx1 = boxes.new()
-    yy1 = boxes.new()
-    xx2 = boxes.new()
-    yy2 = boxes.new()
-    w = boxes.new()
-    h = boxes.new()
-
-    # keep = torch.Tensor()
-    count = 0
-    while idx.numel() > 0:
-        i = idx[-1]  # index of current largest val
-        # keep.append(i)
-        keep[count] = i
-        count += 1
-        if idx.size(0) == 1:
-            break
-        idx = idx[:-1]  # remove kept element from view
-        # load bboxes of next highest vals
-        torch.index_select(x1, 0, idx, out=xx1)
-        torch.index_select(y1, 0, idx, out=yy1)
-        torch.index_select(x2, 0, idx, out=xx2)
-        torch.index_select(y2, 0, idx, out=yy2)
-        # store element-wise max with next highest score
-        xx1 = torch.clamp(xx1, min=x1[i])
-        yy1 = torch.clamp(yy1, min=y1[i])
-        xx2 = torch.clamp(xx2, max=x2[i])
-        yy2 = torch.clamp(yy2, max=y2[i])
-        w.resize_as_(xx2)
-        h.resize_as_(yy2)
-        w = xx2 - xx1
-        h = yy2 - yy1
-        # check sizes of xx1 and xx2.. after each iteration
-        w = torch.clamp(w, min=0.0)
-        h = torch.clamp(h, min=0.0)
-        inter = w * h
-        # IoU = i / (area(a) + area(b) - i)
-        rem_areas = torch.index_select(area, 0, idx)  # load remaining areas)
-        union = (rem_areas - inter) + area[i]
-        IoU = inter / union  # store result in iou
-        # keep only elements with an IoU <= overlap
-        idx = idx[IoU.le(overlap)]
-    return keep, count

extras/facexlib/parsing/__init__.py

@@ -1,24 +0,0 @@
-import torch
-
-from extras.facexlib.utils import load_file_from_url
-from .bisenet import BiSeNet
-from .parsenet import ParseNet
-
-
-def init_parsing_model(model_name='bisenet', half=False, device='cuda', model_rootpath=None):
-    if model_name == 'bisenet':
-        model = BiSeNet(num_class=19)
-        model_url = 'https://github.com/xinntao/facexlib/releases/download/v0.2.0/parsing_bisenet.pth'
-    elif model_name == 'parsenet':
-        model = ParseNet(in_size=512, out_size=512, parsing_ch=19)
-        model_url = 'https://github.com/xinntao/facexlib/releases/download/v0.2.2/parsing_parsenet.pth'
-    else:
-        raise NotImplementedError(f'{model_name} is not implemented.')
-
-    model_path = load_file_from_url(
-        url=model_url, model_dir='facexlib/weights', progress=True, file_name=None, save_dir=model_rootpath)
-    load_net = torch.load(model_path, map_location=lambda storage, loc: storage)
-    model.load_state_dict(load_net, strict=True)
-    model.eval()
-    model = model.to(device)
-    return model

extras/facexlib/parsing/bisenet.py

@@ -1,140 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from .resnet import ResNet18
-
-
-class ConvBNReLU(nn.Module):
-
-    def __init__(self, in_chan, out_chan, ks=3, stride=1, padding=1):
-        super(ConvBNReLU, self).__init__()
-        self.conv = nn.Conv2d(in_chan, out_chan, kernel_size=ks, stride=stride, padding=padding, bias=False)
-        self.bn = nn.BatchNorm2d(out_chan)
-
-    def forward(self, x):
-        x = self.conv(x)
-        x = F.relu(self.bn(x))
-        return x
-
-
-class BiSeNetOutput(nn.Module):
-
-    def __init__(self, in_chan, mid_chan, num_class):
-        super(BiSeNetOutput, self).__init__()
-        self.conv = ConvBNReLU(in_chan, mid_chan, ks=3, stride=1, padding=1)
-        self.conv_out = nn.Conv2d(mid_chan, num_class, kernel_size=1, bias=False)
-
-    def forward(self, x):
-        feat = self.conv(x)
-        out = self.conv_out(feat)
-        return out, feat
-
-
-class AttentionRefinementModule(nn.Module):
-
-    def __init__(self, in_chan, out_chan):
-        super(AttentionRefinementModule, self).__init__()
-        self.conv = ConvBNReLU(in_chan, out_chan, ks=3, stride=1, padding=1)
-        self.conv_atten = nn.Conv2d(out_chan, out_chan, kernel_size=1, bias=False)
-        self.bn_atten = nn.BatchNorm2d(out_chan)
-        self.sigmoid_atten = nn.Sigmoid()
-
-    def forward(self, x):
-        feat = self.conv(x)
-        atten = F.avg_pool2d(feat, feat.size()[2:])
-        atten = self.conv_atten(atten)
-        atten = self.bn_atten(atten)
-        atten = self.sigmoid_atten(atten)
-        out = torch.mul(feat, atten)
-        return out
-
-
-class ContextPath(nn.Module):
-
-    def __init__(self):
-        super(ContextPath, self).__init__()
-        self.resnet = ResNet18()
-        self.arm16 = AttentionRefinementModule(256, 128)
-        self.arm32 = AttentionRefinementModule(512, 128)
-        self.conv_head32 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1)
-        self.conv_head16 = ConvBNReLU(128, 128, ks=3, stride=1, padding=1)
-        self.conv_avg = ConvBNReLU(512, 128, ks=1, stride=1, padding=0)
-
-    def forward(self, x):
-        feat8, feat16, feat32 = self.resnet(x)
-        h8, w8 = feat8.size()[2:]
-        h16, w16 = feat16.size()[2:]
-        h32, w32 = feat32.size()[2:]
-
-        avg = F.avg_pool2d(feat32, feat32.size()[2:])
-        avg = self.conv_avg(avg)
-        avg_up = F.interpolate(avg, (h32, w32), mode='nearest')
-
-        feat32_arm = self.arm32(feat32)
-        feat32_sum = feat32_arm + avg_up
-        feat32_up = F.interpolate(feat32_sum, (h16, w16), mode='nearest')
-        feat32_up = self.conv_head32(feat32_up)
-
-        feat16_arm = self.arm16(feat16)
-        feat16_sum = feat16_arm + feat32_up
-        feat16_up = F.interpolate(feat16_sum, (h8, w8), mode='nearest')
-        feat16_up = self.conv_head16(feat16_up)
-
-        return feat8, feat16_up, feat32_up  # x8, x8, x16
-
-
-class FeatureFusionModule(nn.Module):
-
-    def __init__(self, in_chan, out_chan):
-        super(FeatureFusionModule, self).__init__()
-        self.convblk = ConvBNReLU(in_chan, out_chan, ks=1, stride=1, padding=0)
-        self.conv1 = nn.Conv2d(out_chan, out_chan // 4, kernel_size=1, stride=1, padding=0, bias=False)
-        self.conv2 = nn.Conv2d(out_chan // 4, out_chan, kernel_size=1, stride=1, padding=0, bias=False)
-        self.relu = nn.ReLU(inplace=True)
-        self.sigmoid = nn.Sigmoid()
-
-    def forward(self, fsp, fcp):
-        fcat = torch.cat([fsp, fcp], dim=1)
-        feat = self.convblk(fcat)
-        atten = F.avg_pool2d(feat, feat.size()[2:])
-        atten = self.conv1(atten)
-        atten = self.relu(atten)
-        atten = self.conv2(atten)
-        atten = self.sigmoid(atten)
-        feat_atten = torch.mul(feat, atten)
-        feat_out = feat_atten + feat
-        return feat_out
-
-
-class BiSeNet(nn.Module):
-
-    def __init__(self, num_class):
-        super(BiSeNet, self).__init__()
-        self.cp = ContextPath()
-        self.ffm = FeatureFusionModule(256, 256)
-        self.conv_out = BiSeNetOutput(256, 256, num_class)
-        self.conv_out16 = BiSeNetOutput(128, 64, num_class)
-        self.conv_out32 = BiSeNetOutput(128, 64, num_class)
-
-    def forward(self, x, return_feat=False):
-        h, w = x.size()[2:]
-        feat_res8, feat_cp8, feat_cp16 = self.cp(x)  # return res3b1 feature
-        feat_sp = feat_res8  # replace spatial path feature with res3b1 feature
-        feat_fuse = self.ffm(feat_sp, feat_cp8)
-
-        out, feat = self.conv_out(feat_fuse)
-        out16, feat16 = self.conv_out16(feat_cp8)
-        out32, feat32 = self.conv_out32(feat_cp16)
-
-        out = F.interpolate(out, (h, w), mode='bilinear', align_corners=True)
-        out16 = F.interpolate(out16, (h, w), mode='bilinear', align_corners=True)
-        out32 = F.interpolate(out32, (h, w), mode='bilinear', align_corners=True)
-
-        if return_feat:
-            feat = F.interpolate(feat, (h, w), mode='bilinear', align_corners=True)
-            feat16 = F.interpolate(feat16, (h, w), mode='bilinear', align_corners=True)
-            feat32 = F.interpolate(feat32, (h, w), mode='bilinear', align_corners=True)
-            return out, out16, out32, feat, feat16, feat32
-        else:
-            return out, out16, out32

extras/facexlib/parsing/parsenet.py

@@ -1,194 +0,0 @@
-"""Modified from https://github.com/chaofengc/PSFRGAN
-"""
-import numpy as np
-import torch.nn as nn
-from torch.nn import functional as F
-
-
-class NormLayer(nn.Module):
-    """Normalization Layers.
-
-    Args:
-        channels: input channels, for batch norm and instance norm.
-        input_size: input shape without batch size, for layer norm.
-    """
-
-    def __init__(self, channels, normalize_shape=None, norm_type='bn'):
-        super(NormLayer, self).__init__()
-        norm_type = norm_type.lower()
-        self.norm_type = norm_type
-        if norm_type == 'bn':
-            self.norm = nn.BatchNorm2d(channels, affine=True)
-        elif norm_type == 'in':
-            self.norm = nn.InstanceNorm2d(channels, affine=False)
-        elif norm_type == 'gn':
-            self.norm = nn.GroupNorm(32, channels, affine=True)
-        elif norm_type == 'pixel':
-            self.norm = lambda x: F.normalize(x, p=2, dim=1)
-        elif norm_type == 'layer':
-            self.norm = nn.LayerNorm(normalize_shape)
-        elif norm_type == 'none':
-            self.norm = lambda x: x * 1.0
-        else:
-            assert 1 == 0, f'Norm type {norm_type} not support.'
-
-    def forward(self, x, ref=None):
-        if self.norm_type == 'spade':
-            return self.norm(x, ref)
-        else:
-            return self.norm(x)
-
-
-class ReluLayer(nn.Module):
-    """Relu Layer.
-
-    Args:
-        relu type: type of relu layer, candidates are
-            - ReLU
-            - LeakyReLU: default relu slope 0.2
-            - PRelu
-            - SELU
-            - none: direct pass
-    """
-
-    def __init__(self, channels, relu_type='relu'):
-        super(ReluLayer, self).__init__()
-        relu_type = relu_type.lower()
-        if relu_type == 'relu':
-            self.func = nn.ReLU(True)
-        elif relu_type == 'leakyrelu':
-            self.func = nn.LeakyReLU(0.2, inplace=True)
-        elif relu_type == 'prelu':
-            self.func = nn.PReLU(channels)
-        elif relu_type == 'selu':
-            self.func = nn.SELU(True)
-        elif relu_type == 'none':
-            self.func = lambda x: x * 1.0
-        else:
-            assert 1 == 0, f'Relu type {relu_type} not support.'
-
-    def forward(self, x):
-        return self.func(x)
-
-
-class ConvLayer(nn.Module):
-
-    def __init__(self,
-                 in_channels,
-                 out_channels,
-                 kernel_size=3,
-                 scale='none',
-                 norm_type='none',
-                 relu_type='none',
-                 use_pad=True,
-                 bias=True):
-        super(ConvLayer, self).__init__()
-        self.use_pad = use_pad
-        self.norm_type = norm_type
-        if norm_type in ['bn']:
-            bias = False
-
-        stride = 2 if scale == 'down' else 1
-
-        self.scale_func = lambda x: x
-        if scale == 'up':
-            self.scale_func = lambda x: nn.functional.interpolate(x, scale_factor=2, mode='nearest')
-
-        self.reflection_pad = nn.ReflectionPad2d(int(np.ceil((kernel_size - 1.) / 2)))
-        self.conv2d = nn.Conv2d(in_channels, out_channels, kernel_size, stride, bias=bias)
-
-        self.relu = ReluLayer(out_channels, relu_type)
-        self.norm = NormLayer(out_channels, norm_type=norm_type)
-
-    def forward(self, x):
-        out = self.scale_func(x)
-        if self.use_pad:
-            out = self.reflection_pad(out)
-        out = self.conv2d(out)
-        out = self.norm(out)
-        out = self.relu(out)
-        return out
-
-
-class ResidualBlock(nn.Module):
-    """
-    Residual block recommended in: http://torch.ch/blog/2016/02/04/resnets.html
-    """
-
-    def __init__(self, c_in, c_out, relu_type='prelu', norm_type='bn', scale='none'):
-        super(ResidualBlock, self).__init__()
-
-        if scale == 'none' and c_in == c_out:
-            self.shortcut_func = lambda x: x
-        else:
-            self.shortcut_func = ConvLayer(c_in, c_out, 3, scale)
-
-        scale_config_dict = {'down': ['none', 'down'], 'up': ['up', 'none'], 'none': ['none', 'none']}
-        scale_conf = scale_config_dict[scale]
-
-        self.conv1 = ConvLayer(c_in, c_out, 3, scale_conf[0], norm_type=norm_type, relu_type=relu_type)
-        self.conv2 = ConvLayer(c_out, c_out, 3, scale_conf[1], norm_type=norm_type, relu_type='none')
-
-    def forward(self, x):
-        identity = self.shortcut_func(x)
-
-        res = self.conv1(x)
-        res = self.conv2(res)
-        return identity + res
-
-
-class ParseNet(nn.Module):
-
-    def __init__(self,
-                 in_size=128,
-                 out_size=128,
-                 min_feat_size=32,
-                 base_ch=64,
-                 parsing_ch=19,
-                 res_depth=10,
-                 relu_type='LeakyReLU',
-                 norm_type='bn',
-                 ch_range=[32, 256]):
-        super().__init__()
-        self.res_depth = res_depth
-        act_args = {'norm_type': norm_type, 'relu_type': relu_type}
-        min_ch, max_ch = ch_range
-
-        ch_clip = lambda x: max(min_ch, min(x, max_ch))  # noqa: E731
-        min_feat_size = min(in_size, min_feat_size)
-
-        down_steps = int(np.log2(in_size // min_feat_size))
-        up_steps = int(np.log2(out_size // min_feat_size))
-
-        # =============== define encoder-body-decoder ====================
-        self.encoder = []
-        self.encoder.append(ConvLayer(3, base_ch, 3, 1))
-        head_ch = base_ch
-        for i in range(down_steps):
-            cin, cout = ch_clip(head_ch), ch_clip(head_ch * 2)
-            self.encoder.append(ResidualBlock(cin, cout, scale='down', **act_args))
-            head_ch = head_ch * 2
-
-        self.body = []
-        for i in range(res_depth):
-            self.body.append(ResidualBlock(ch_clip(head_ch), ch_clip(head_ch), **act_args))
-
-        self.decoder = []
-        for i in range(up_steps):
-            cin, cout = ch_clip(head_ch), ch_clip(head_ch // 2)
-            self.decoder.append(ResidualBlock(cin, cout, scale='up', **act_args))
-            head_ch = head_ch // 2
-
-        self.encoder = nn.Sequential(*self.encoder)
-        self.body = nn.Sequential(*self.body)
-        self.decoder = nn.Sequential(*self.decoder)
-        self.out_img_conv = ConvLayer(ch_clip(head_ch), 3)
-        self.out_mask_conv = ConvLayer(ch_clip(head_ch), parsing_ch)
-
-    def forward(self, x):
-        feat = self.encoder(x)
-        x = feat + self.body(feat)
-        x = self.decoder(x)
-        out_img = self.out_img_conv(x)
-        out_mask = self.out_mask_conv(x)
-        return out_mask, out_img

extras/facexlib/parsing/resnet.py

@@ -1,69 +0,0 @@
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-def conv3x3(in_planes, out_planes, stride=1):
-    """3x3 convolution with padding"""
-    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False)
-
-
-class BasicBlock(nn.Module):
-
-    def __init__(self, in_chan, out_chan, stride=1):
-        super(BasicBlock, self).__init__()
-        self.conv1 = conv3x3(in_chan, out_chan, stride)
-        self.bn1 = nn.BatchNorm2d(out_chan)
-        self.conv2 = conv3x3(out_chan, out_chan)
-        self.bn2 = nn.BatchNorm2d(out_chan)
-        self.relu = nn.ReLU(inplace=True)
-        self.downsample = None
-        if in_chan != out_chan or stride != 1:
-            self.downsample = nn.Sequential(
-                nn.Conv2d(in_chan, out_chan, kernel_size=1, stride=stride, bias=False),
-                nn.BatchNorm2d(out_chan),
-            )
-
-    def forward(self, x):
-        residual = self.conv1(x)
-        residual = F.relu(self.bn1(residual))
-        residual = self.conv2(residual)
-        residual = self.bn2(residual)
-
-        shortcut = x
-        if self.downsample is not None:
-            shortcut = self.downsample(x)
-
-        out = shortcut + residual
-        out = self.relu(out)
-        return out
-
-
-def create_layer_basic(in_chan, out_chan, bnum, stride=1):
-    layers = [BasicBlock(in_chan, out_chan, stride=stride)]
-    for i in range(bnum - 1):
-        layers.append(BasicBlock(out_chan, out_chan, stride=1))
-    return nn.Sequential(*layers)
-
-
-class ResNet18(nn.Module):
-
-    def __init__(self):
-        super(ResNet18, self).__init__()
-        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
-        self.bn1 = nn.BatchNorm2d(64)
-        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
-        self.layer1 = create_layer_basic(64, 64, bnum=2, stride=1)
-        self.layer2 = create_layer_basic(64, 128, bnum=2, stride=2)
-        self.layer3 = create_layer_basic(128, 256, bnum=2, stride=2)
-        self.layer4 = create_layer_basic(256, 512, bnum=2, stride=2)
-
-    def forward(self, x):
-        x = self.conv1(x)
-        x = F.relu(self.bn1(x))
-        x = self.maxpool(x)
-
-        x = self.layer1(x)
-        feat8 = self.layer2(x)  # 1/8
-        feat16 = self.layer3(feat8)  # 1/16
-        feat32 = self.layer4(feat16)  # 1/32
-        return feat8, feat16, feat32

extras/facexlib/utils/__init__.py

@@ -1,7 +0,0 @@
-from .face_utils import align_crop_face_landmarks, compute_increased_bbox, get_valid_bboxes, paste_face_back
-from .misc import img2tensor, load_file_from_url, scandir
-
-__all__ = [
-    'align_crop_face_landmarks', 'compute_increased_bbox', 'get_valid_bboxes', 'load_file_from_url', 'paste_face_back',
-    'img2tensor', 'scandir'
-]

extras/facexlib/utils/face_restoration_helper.py

@@ -1,374 +0,0 @@
-import cv2
-import numpy as np
-import os
-import torch
-from torchvision.transforms.functional import normalize
-
-from extras.facexlib.detection import init_detection_model
-from extras.facexlib.parsing import init_parsing_model
-from extras.facexlib.utils.misc import img2tensor, imwrite
-
-
-def get_largest_face(det_faces, h, w):
-
-    def get_location(val, length):
-        if val < 0:
-            return 0
-        elif val > length:
-            return length
-        else:
-            return val
-
-    face_areas = []
-    for det_face in det_faces:
-        left = get_location(det_face[0], w)
-        right = get_location(det_face[2], w)
-        top = get_location(det_face[1], h)
-        bottom = get_location(det_face[3], h)
-        face_area = (right - left) * (bottom - top)
-        face_areas.append(face_area)
-    largest_idx = face_areas.index(max(face_areas))
-    return det_faces[largest_idx], largest_idx
-
-
-def get_center_face(det_faces, h=0, w=0, center=None):
-    if center is not None:
-        center = np.array(center)
-    else:
-        center = np.array([w / 2, h / 2])
-    center_dist = []
-    for det_face in det_faces:
-        face_center = np.array([(det_face[0] + det_face[2]) / 2, (det_face[1] + det_face[3]) / 2])
-        dist = np.linalg.norm(face_center - center)
-        center_dist.append(dist)
-    center_idx = center_dist.index(min(center_dist))
-    return det_faces[center_idx], center_idx
-
-
-class FaceRestoreHelper(object):
-    """Helper for the face restoration pipeline (base class)."""
-
-    def __init__(self,
-                 upscale_factor,
-                 face_size=512,
-                 crop_ratio=(1, 1),
-                 det_model='retinaface_resnet50',
-                 save_ext='png',
-                 template_3points=False,
-                 pad_blur=False,
-                 use_parse=False,
-                 device=None,
-                 model_rootpath=None):
-        self.template_3points = template_3points  # improve robustness
-        self.upscale_factor = upscale_factor
-        # the cropped face ratio based on the square face
-        self.crop_ratio = crop_ratio  # (h, w)
-        assert (self.crop_ratio[0] >= 1 and self.crop_ratio[1] >= 1), 'crop ration only supports >=1'
-        self.face_size = (int(face_size * self.crop_ratio[1]), int(face_size * self.crop_ratio[0]))
-
-        if self.template_3points:
-            self.face_template = np.array([[192, 240], [319, 240], [257, 371]])
-        else:
-            # standard 5 landmarks for FFHQ faces with 512 x 512
-            self.face_template = np.array([[192.98138, 239.94708], [318.90277, 240.1936], [256.63416, 314.01935],
-                                           [201.26117, 371.41043], [313.08905, 371.15118]])
-        self.face_template = self.face_template * (face_size / 512.0)
-        if self.crop_ratio[0] > 1:
-            self.face_template[:, 1] += face_size * (self.crop_ratio[0] - 1) / 2
-        if self.crop_ratio[1] > 1:
-            self.face_template[:, 0] += face_size * (self.crop_ratio[1] - 1) / 2
-        self.save_ext = save_ext
-        self.pad_blur = pad_blur
-        if self.pad_blur is True:
-            self.template_3points = False
-
-        self.all_landmarks_5 = []
-        self.det_faces = []
-        self.affine_matrices = []
-        self.inverse_affine_matrices = []
-        self.cropped_faces = []
-        self.restored_faces = []
-        self.pad_input_imgs = []
-
-        if device is None:
-            self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        else:
-            self.device = device
-
-        # init face detection model
-        self.face_det = init_detection_model(det_model, half=False, device=self.device, model_rootpath=model_rootpath)
-
-        # init face parsing model
-        self.use_parse = use_parse
-        self.face_parse = init_parsing_model(model_name='parsenet', device=self.device, model_rootpath=model_rootpath)
-
-    def set_upscale_factor(self, upscale_factor):
-        self.upscale_factor = upscale_factor
-
-    def read_image(self, img):
-        """img can be image path or cv2 loaded image."""
-        # self.input_img is Numpy array, (h, w, c), BGR, uint8, [0, 255]
-        if isinstance(img, str):
-            img = cv2.imread(img)
-
-        if np.max(img) > 256:  # 16-bit image
-            img = img / 65535 * 255
-        if len(img.shape) == 2:  # gray image
-            img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
-        elif img.shape[2] == 4:  # RGBA image with alpha channel
-            img = img[:, :, 0:3]
-
-        self.input_img = img
-
-    def get_face_landmarks_5(self,
-                             only_keep_largest=False,
-                             only_center_face=False,
-                             resize=None,
-                             blur_ratio=0.01,
-                             eye_dist_threshold=None):
-        if resize is None:
-            scale = 1
-            input_img = self.input_img
-        else:
-            h, w = self.input_img.shape[0:2]
-            scale = min(h, w) / resize
-            h, w = int(h / scale), int(w / scale)
-            input_img = cv2.resize(self.input_img, (w, h), interpolation=cv2.INTER_LANCZOS4)
-
-        with torch.no_grad():
-            bboxes = self.face_det.detect_faces(input_img, 0.97) * scale
-        for bbox in bboxes:
-            # remove faces with too small eye distance: side faces or too small faces
-            eye_dist = np.linalg.norm([bbox[5] - bbox[7], bbox[6] - bbox[8]])
-            if eye_dist_threshold is not None and (eye_dist < eye_dist_threshold):
-                continue
-
-            if self.template_3points:
-                landmark = np.array([[bbox[i], bbox[i + 1]] for i in range(5, 11, 2)])
-            else:
-                landmark = np.array([[bbox[i], bbox[i + 1]] for i in range(5, 15, 2)])
-            self.all_landmarks_5.append(landmark)
-            self.det_faces.append(bbox[0:5])
-        if len(self.det_faces) == 0:
-            return 0
-        if only_keep_largest:
-            h, w, _ = self.input_img.shape
-            self.det_faces, largest_idx = get_largest_face(self.det_faces, h, w)
-            self.all_landmarks_5 = [self.all_landmarks_5[largest_idx]]
-        elif only_center_face:
-            h, w, _ = self.input_img.shape
-            self.det_faces, center_idx = get_center_face(self.det_faces, h, w)
-            self.all_landmarks_5 = [self.all_landmarks_5[center_idx]]
-
-        # pad blurry images
-        if self.pad_blur:
-            self.pad_input_imgs = []
-            for landmarks in self.all_landmarks_5:
-                # get landmarks
-                eye_left = landmarks[0, :]
-                eye_right = landmarks[1, :]
-                eye_avg = (eye_left + eye_right) * 0.5
-                mouth_avg = (landmarks[3, :] + landmarks[4, :]) * 0.5
-                eye_to_eye = eye_right - eye_left
-                eye_to_mouth = mouth_avg - eye_avg
-
-                # Get the oriented crop rectangle
-                # x: half width of the oriented crop rectangle
-                x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1]
-                #  - np.flipud(eye_to_mouth) * [-1, 1]: rotate 90 clockwise
-                # norm with the hypotenuse: get the direction
-                x /= np.hypot(*x)  # get the hypotenuse of a right triangle
-                rect_scale = 1.5
-                x *= max(np.hypot(*eye_to_eye) * 2.0 * rect_scale, np.hypot(*eye_to_mouth) * 1.8 * rect_scale)
-                # y: half height of the oriented crop rectangle
-                y = np.flipud(x) * [-1, 1]
-
-                # c: center
-                c = eye_avg + eye_to_mouth * 0.1
-                # quad: (left_top, left_bottom, right_bottom, right_top)
-                quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y])
-                # qsize: side length of the square
-                qsize = np.hypot(*x) * 2
-                border = max(int(np.rint(qsize * 0.1)), 3)
-
-                # get pad
-                # pad: (width_left, height_top, width_right, height_bottom)
-                pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),
-                       int(np.ceil(max(quad[:, 1]))))
-                pad = [
-                    max(-pad[0] + border, 1),
-                    max(-pad[1] + border, 1),
-                    max(pad[2] - self.input_img.shape[0] + border, 1),
-                    max(pad[3] - self.input_img.shape[1] + border, 1)
-                ]
-
-                if max(pad) > 1:
-                    # pad image
-                    pad_img = np.pad(self.input_img, ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect')
-                    # modify landmark coords
-                    landmarks[:, 0] += pad[0]
-                    landmarks[:, 1] += pad[1]
-                    # blur pad images
-                    h, w, _ = pad_img.shape
-                    y, x, _ = np.ogrid[:h, :w, :1]
-                    mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0],
-                                                       np.float32(w - 1 - x) / pad[2]),
-                                      1.0 - np.minimum(np.float32(y) / pad[1],
-                                                       np.float32(h - 1 - y) / pad[3]))
-                    blur = int(qsize * blur_ratio)
-                    if blur % 2 == 0:
-                        blur += 1
-                    blur_img = cv2.boxFilter(pad_img, 0, ksize=(blur, blur))
-                    # blur_img = cv2.GaussianBlur(pad_img, (blur, blur), 0)
-
-                    pad_img = pad_img.astype('float32')
-                    pad_img += (blur_img - pad_img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0)
-                    pad_img += (np.median(pad_img, axis=(0, 1)) - pad_img) * np.clip(mask, 0.0, 1.0)
-                    pad_img = np.clip(pad_img, 0, 255)  # float32, [0, 255]
-                    self.pad_input_imgs.append(pad_img)
-                else:
-                    self.pad_input_imgs.append(np.copy(self.input_img))
-
-        return len(self.all_landmarks_5)
-
-    def align_warp_face(self, save_cropped_path=None, border_mode='constant'):
-        """Align and warp faces with face template.
-        """
-        if self.pad_blur:
-            assert len(self.pad_input_imgs) == len(
-                self.all_landmarks_5), f'Mismatched samples: {len(self.pad_input_imgs)} and {len(self.all_landmarks_5)}'
-        for idx, landmark in enumerate(self.all_landmarks_5):
-            # use 5 landmarks to get affine matrix
-            # use cv2.LMEDS method for the equivalence to skimage transform
-            # ref: https://blog.csdn.net/yichxi/article/details/115827338
-            affine_matrix = cv2.estimateAffinePartial2D(landmark, self.face_template, method=cv2.LMEDS)[0]
-            self.affine_matrices.append(affine_matrix)
-            # warp and crop faces
-            if border_mode == 'constant':
-                border_mode = cv2.BORDER_CONSTANT
-            elif border_mode == 'reflect101':
-                border_mode = cv2.BORDER_REFLECT101
-            elif border_mode == 'reflect':
-                border_mode = cv2.BORDER_REFLECT
-            if self.pad_blur:
-                input_img = self.pad_input_imgs[idx]
-            else:
-                input_img = self.input_img
-            cropped_face = cv2.warpAffine(
-                input_img, affine_matrix, self.face_size, borderMode=border_mode, borderValue=(135, 133, 132))  # gray
-            self.cropped_faces.append(cropped_face)
-            # save the cropped face
-            if save_cropped_path is not None:
-                path = os.path.splitext(save_cropped_path)[0]
-                save_path = f'{path}_{idx:02d}.{self.save_ext}'
-                imwrite(cropped_face, save_path)
-
-    def get_inverse_affine(self, save_inverse_affine_path=None):
-        """Get inverse affine matrix."""
-        for idx, affine_matrix in enumerate(self.affine_matrices):
-            inverse_affine = cv2.invertAffineTransform(affine_matrix)
-            inverse_affine *= self.upscale_factor
-            self.inverse_affine_matrices.append(inverse_affine)
-            # save inverse affine matrices
-            if save_inverse_affine_path is not None:
-                path, _ = os.path.splitext(save_inverse_affine_path)
-                save_path = f'{path}_{idx:02d}.pth'
-                torch.save(inverse_affine, save_path)
-
-    def add_restored_face(self, face):
-        self.restored_faces.append(face)
-
-    def paste_faces_to_input_image(self, save_path=None, upsample_img=None):
-        h, w, _ = self.input_img.shape
-        h_up, w_up = int(h * self.upscale_factor), int(w * self.upscale_factor)
-
-        if upsample_img is None:
-            # simply resize the background
-            upsample_img = cv2.resize(self.input_img, (w_up, h_up), interpolation=cv2.INTER_LANCZOS4)
-        else:
-            upsample_img = cv2.resize(upsample_img, (w_up, h_up), interpolation=cv2.INTER_LANCZOS4)
-
-        assert len(self.restored_faces) == len(
-            self.inverse_affine_matrices), ('length of restored_faces and affine_matrices are different.')
-        for restored_face, inverse_affine in zip(self.restored_faces, self.inverse_affine_matrices):
-            # Add an offset to inverse affine matrix, for more precise back alignment
-            if self.upscale_factor > 1:
-                extra_offset = 0.5 * self.upscale_factor
-            else:
-                extra_offset = 0
-            inverse_affine[:, 2] += extra_offset
-            inv_restored = cv2.warpAffine(restored_face, inverse_affine, (w_up, h_up))
-
-            if self.use_parse:
-                # inference
-                face_input = cv2.resize(restored_face, (512, 512), interpolation=cv2.INTER_LINEAR)
-                face_input = img2tensor(face_input.astype('float32') / 255., bgr2rgb=True, float32=True)
-                normalize(face_input, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
-                face_input = torch.unsqueeze(face_input, 0).to(self.device)
-                with torch.no_grad():
-                    out = self.face_parse(face_input)[0]
-                out = out.argmax(dim=1).squeeze().cpu().numpy()
-
-                mask = np.zeros(out.shape)
-                MASK_COLORMAP = [0, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 255, 0, 0, 0]
-                for idx, color in enumerate(MASK_COLORMAP):
-                    mask[out == idx] = color
-                #  blur the mask
-                mask = cv2.GaussianBlur(mask, (101, 101), 11)
-                mask = cv2.GaussianBlur(mask, (101, 101), 11)
-                # remove the black borders
-                thres = 10
-                mask[:thres, :] = 0
-                mask[-thres:, :] = 0
-                mask[:, :thres] = 0
-                mask[:, -thres:] = 0
-                mask = mask / 255.
-
-                mask = cv2.resize(mask, restored_face.shape[:2])
-                mask = cv2.warpAffine(mask, inverse_affine, (w_up, h_up), flags=3)
-                inv_soft_mask = mask[:, :, None]
-                pasted_face = inv_restored
-
-            else:  # use square parse maps
-                mask = np.ones(self.face_size, dtype=np.float32)
-                inv_mask = cv2.warpAffine(mask, inverse_affine, (w_up, h_up))
-                # remove the black borders
-                inv_mask_erosion = cv2.erode(
-                    inv_mask, np.ones((int(2 * self.upscale_factor), int(2 * self.upscale_factor)), np.uint8))
-                pasted_face = inv_mask_erosion[:, :, None] * inv_restored
-                total_face_area = np.sum(inv_mask_erosion)  # // 3
-                # compute the fusion edge based on the area of face
-                w_edge = int(total_face_area**0.5) // 20
-                erosion_radius = w_edge * 2
-                inv_mask_center = cv2.erode(inv_mask_erosion, np.ones((erosion_radius, erosion_radius), np.uint8))
-                blur_size = w_edge * 2
-                inv_soft_mask = cv2.GaussianBlur(inv_mask_center, (blur_size + 1, blur_size + 1), 0)
-                if len(upsample_img.shape) == 2:  # upsample_img is gray image
-                    upsample_img = upsample_img[:, :, None]
-                inv_soft_mask = inv_soft_mask[:, :, None]
-
-            if len(upsample_img.shape) == 3 and upsample_img.shape[2] == 4:  # alpha channel
-                alpha = upsample_img[:, :, 3:]
-                upsample_img = inv_soft_mask * pasted_face + (1 - inv_soft_mask) * upsample_img[:, :, 0:3]
-                upsample_img = np.concatenate((upsample_img, alpha), axis=2)
-            else:
-                upsample_img = inv_soft_mask * pasted_face + (1 - inv_soft_mask) * upsample_img
-
-        if np.max(upsample_img) > 256:  # 16-bit image
-            upsample_img = upsample_img.astype(np.uint16)
-        else:
-            upsample_img = upsample_img.astype(np.uint8)
-        if save_path is not None:
-            path = os.path.splitext(save_path)[0]
-            save_path = f'{path}.{self.save_ext}'
-            imwrite(upsample_img, save_path)
-        return upsample_img
-
-    def clean_all(self):
-        self.all_landmarks_5 = []
-        self.restored_faces = []
-        self.affine_matrices = []
-        self.cropped_faces = []
-        self.inverse_affine_matrices = []
-        self.det_faces = []
-        self.pad_input_imgs = []

extras/facexlib/utils/face_utils.py

@@ -1,250 +0,0 @@
-import cv2
-import numpy as np
-import torch
-
-
-def compute_increased_bbox(bbox, increase_area, preserve_aspect=True):
-    left, top, right, bot = bbox
-    width = right - left
-    height = bot - top
-
-    if preserve_aspect:
-        width_increase = max(increase_area, ((1 + 2 * increase_area) * height - width) / (2 * width))
-        height_increase = max(increase_area, ((1 + 2 * increase_area) * width - height) / (2 * height))
-    else:
-        width_increase = height_increase = increase_area
-    left = int(left - width_increase * width)
-    top = int(top - height_increase * height)
-    right = int(right + width_increase * width)
-    bot = int(bot + height_increase * height)
-    return (left, top, right, bot)
-
-
-def get_valid_bboxes(bboxes, h, w):
-    left = max(bboxes[0], 0)
-    top = max(bboxes[1], 0)
-    right = min(bboxes[2], w)
-    bottom = min(bboxes[3], h)
-    return (left, top, right, bottom)
-
-
-def align_crop_face_landmarks(img,
-                              landmarks,
-                              output_size,
-                              transform_size=None,
-                              enable_padding=True,
-                              return_inverse_affine=False,
-                              shrink_ratio=(1, 1)):
-    """Align and crop face with landmarks.
-
-    The output_size and transform_size are based on width. The height is
-    adjusted based on shrink_ratio_h/shring_ration_w.
-
-    Modified from:
-    https://github.com/NVlabs/ffhq-dataset/blob/master/download_ffhq.py
-
-    Args:
-        img (Numpy array): Input image.
-        landmarks (Numpy array): 5 or 68 or 98 landmarks.
-        output_size (int): Output face size.
-        transform_size (ing): Transform size. Usually the four time of
-            output_size.
-        enable_padding (float): Default: True.
-        shrink_ratio (float | tuple[float] | list[float]): Shring the whole
-            face for height and width (crop larger area). Default: (1, 1).
-
-    Returns:
-        (Numpy array): Cropped face.
-    """
-    lm_type = 'retinaface_5'  # Options: dlib_5, retinaface_5
-
-    if isinstance(shrink_ratio, (float, int)):
-        shrink_ratio = (shrink_ratio, shrink_ratio)
-    if transform_size is None:
-        transform_size = output_size * 4
-
-    # Parse landmarks
-    lm = np.array(landmarks)
-    if lm.shape[0] == 5 and lm_type == 'retinaface_5':
-        eye_left = lm[0]
-        eye_right = lm[1]
-        mouth_avg = (lm[3] + lm[4]) * 0.5
-    elif lm.shape[0] == 5 and lm_type == 'dlib_5':
-        lm_eye_left = lm[2:4]
-        lm_eye_right = lm[0:2]
-        eye_left = np.mean(lm_eye_left, axis=0)
-        eye_right = np.mean(lm_eye_right, axis=0)
-        mouth_avg = lm[4]
-    elif lm.shape[0] == 68:
-        lm_eye_left = lm[36:42]
-        lm_eye_right = lm[42:48]
-        eye_left = np.mean(lm_eye_left, axis=0)
-        eye_right = np.mean(lm_eye_right, axis=0)
-        mouth_avg = (lm[48] + lm[54]) * 0.5
-    elif lm.shape[0] == 98:
-        lm_eye_left = lm[60:68]
-        lm_eye_right = lm[68:76]
-        eye_left = np.mean(lm_eye_left, axis=0)
-        eye_right = np.mean(lm_eye_right, axis=0)
-        mouth_avg = (lm[76] + lm[82]) * 0.5
-
-    eye_avg = (eye_left + eye_right) * 0.5
-    eye_to_eye = eye_right - eye_left
-    eye_to_mouth = mouth_avg - eye_avg
-
-    # Get the oriented crop rectangle
-    # x: half width of the oriented crop rectangle
-    x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1]
-    #  - np.flipud(eye_to_mouth) * [-1, 1]: rotate 90 clockwise
-    # norm with the hypotenuse: get the direction
-    x /= np.hypot(*x)  # get the hypotenuse of a right triangle
-    rect_scale = 1  # TODO: you can edit it to get larger rect
-    x *= max(np.hypot(*eye_to_eye) * 2.0 * rect_scale, np.hypot(*eye_to_mouth) * 1.8 * rect_scale)
-    # y: half height of the oriented crop rectangle
-    y = np.flipud(x) * [-1, 1]
-
-    x *= shrink_ratio[1]  # width
-    y *= shrink_ratio[0]  # height
-
-    # c: center
-    c = eye_avg + eye_to_mouth * 0.1
-    # quad: (left_top, left_bottom, right_bottom, right_top)
-    quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y])
-    # qsize: side length of the square
-    qsize = np.hypot(*x) * 2
-
-    quad_ori = np.copy(quad)
-    # Shrink, for large face
-    # TODO: do we really need shrink
-    shrink = int(np.floor(qsize / output_size * 0.5))
-    if shrink > 1:
-        h, w = img.shape[0:2]
-        rsize = (int(np.rint(float(w) / shrink)), int(np.rint(float(h) / shrink)))
-        img = cv2.resize(img, rsize, interpolation=cv2.INTER_AREA)
-        quad /= shrink
-        qsize /= shrink
-
-    # Crop
-    h, w = img.shape[0:2]
-    border = max(int(np.rint(qsize * 0.1)), 3)
-    crop = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),
-            int(np.ceil(max(quad[:, 1]))))
-    crop = (max(crop[0] - border, 0), max(crop[1] - border, 0), min(crop[2] + border, w), min(crop[3] + border, h))
-    if crop[2] - crop[0] < w or crop[3] - crop[1] < h:
-        img = img[crop[1]:crop[3], crop[0]:crop[2], :]
-        quad -= crop[0:2]
-
-    # Pad
-    # pad: (width_left, height_top, width_right, height_bottom)
-    h, w = img.shape[0:2]
-    pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),
-           int(np.ceil(max(quad[:, 1]))))
-    pad = (max(-pad[0] + border, 0), max(-pad[1] + border, 0), max(pad[2] - w + border, 0), max(pad[3] - h + border, 0))
-    if enable_padding and max(pad) > border - 4:
-        pad = np.maximum(pad, int(np.rint(qsize * 0.3)))
-        img = np.pad(img, ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect')
-        h, w = img.shape[0:2]
-        y, x, _ = np.ogrid[:h, :w, :1]
-        mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0],
-                                           np.float32(w - 1 - x) / pad[2]),
-                          1.0 - np.minimum(np.float32(y) / pad[1],
-                                           np.float32(h - 1 - y) / pad[3]))
-        blur = int(qsize * 0.02)
-        if blur % 2 == 0:
-            blur += 1
-        blur_img = cv2.boxFilter(img, 0, ksize=(blur, blur))
-
-        img = img.astype('float32')
-        img += (blur_img - img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0)
-        img += (np.median(img, axis=(0, 1)) - img) * np.clip(mask, 0.0, 1.0)
-        img = np.clip(img, 0, 255)  # float32, [0, 255]
-        quad += pad[:2]
-
-    # Transform use cv2
-    h_ratio = shrink_ratio[0] / shrink_ratio[1]
-    dst_h, dst_w = int(transform_size * h_ratio), transform_size
-    template = np.array([[0, 0], [0, dst_h], [dst_w, dst_h], [dst_w, 0]])
-    # use cv2.LMEDS method for the equivalence to skimage transform
-    # ref: https://blog.csdn.net/yichxi/article/details/115827338
-    affine_matrix = cv2.estimateAffinePartial2D(quad, template, method=cv2.LMEDS)[0]
-    cropped_face = cv2.warpAffine(
-        img, affine_matrix, (dst_w, dst_h), borderMode=cv2.BORDER_CONSTANT, borderValue=(135, 133, 132))  # gray
-
-    if output_size < transform_size:
-        cropped_face = cv2.resize(
-            cropped_face, (output_size, int(output_size * h_ratio)), interpolation=cv2.INTER_LINEAR)
-
-    if return_inverse_affine:
-        dst_h, dst_w = int(output_size * h_ratio), output_size
-        template = np.array([[0, 0], [0, dst_h], [dst_w, dst_h], [dst_w, 0]])
-        # use cv2.LMEDS method for the equivalence to skimage transform
-        # ref: https://blog.csdn.net/yichxi/article/details/115827338
-        affine_matrix = cv2.estimateAffinePartial2D(
-            quad_ori, np.array([[0, 0], [0, output_size], [dst_w, dst_h], [dst_w, 0]]), method=cv2.LMEDS)[0]
-        inverse_affine = cv2.invertAffineTransform(affine_matrix)
-    else:
-        inverse_affine = None
-    return cropped_face, inverse_affine
-
-
-def paste_face_back(img, face, inverse_affine):
-    h, w = img.shape[0:2]
-    face_h, face_w = face.shape[0:2]
-    inv_restored = cv2.warpAffine(face, inverse_affine, (w, h))
-    mask = np.ones((face_h, face_w, 3), dtype=np.float32)
-    inv_mask = cv2.warpAffine(mask, inverse_affine, (w, h))
-    # remove the black borders
-    inv_mask_erosion = cv2.erode(inv_mask, np.ones((2, 2), np.uint8))
-    inv_restored_remove_border = inv_mask_erosion * inv_restored
-    total_face_area = np.sum(inv_mask_erosion) // 3
-    # compute the fusion edge based on the area of face
-    w_edge = int(total_face_area**0.5) // 20
-    erosion_radius = w_edge * 2
-    inv_mask_center = cv2.erode(inv_mask_erosion, np.ones((erosion_radius, erosion_radius), np.uint8))
-    blur_size = w_edge * 2
-    inv_soft_mask = cv2.GaussianBlur(inv_mask_center, (blur_size + 1, blur_size + 1), 0)
-    img = inv_soft_mask * inv_restored_remove_border + (1 - inv_soft_mask) * img
-    # float32, [0, 255]
-    return img
-
-
-if __name__ == '__main__':
-    import os
-
-    from extras.facexlib.detection import init_detection_model
-    from extras.facexlib.utils.face_restoration_helper import get_largest_face
-    from extras.facexlib.visualization import visualize_detection
-
-    img_path = '/home/wxt/datasets/ffhq/ffhq_wild/00009.png'
-    img_name = os.splitext(os.path.basename(img_path))[0]
-
-    # initialize model
-    det_net = init_detection_model('retinaface_resnet50', half=False)
-    img_ori = cv2.imread(img_path)
-    h, w = img_ori.shape[0:2]
-    # if larger than 800, scale it
-    scale = max(h / 800, w / 800)
-    if scale > 1:
-        img = cv2.resize(img_ori, (int(w / scale), int(h / scale)), interpolation=cv2.INTER_LINEAR)
-
-    with torch.no_grad():
-        bboxes = det_net.detect_faces(img, 0.97)
-    if scale > 1:
-        bboxes *= scale  # the score is incorrect
-    bboxes = get_largest_face(bboxes, h, w)[0]
-    visualize_detection(img_ori, [bboxes], f'tmp/{img_name}_det.png')
-
-    landmarks = np.array([[bboxes[i], bboxes[i + 1]] for i in range(5, 15, 2)])
-
-    cropped_face, inverse_affine = align_crop_face_landmarks(
-        img_ori,
-        landmarks,
-        output_size=512,
-        transform_size=None,
-        enable_padding=True,
-        return_inverse_affine=True,
-        shrink_ratio=(1, 1))
-
-    cv2.imwrite(f'tmp/{img_name}_cropeed_face.png', cropped_face)
-    img = paste_face_back(img_ori, cropped_face, inverse_affine)
-    cv2.imwrite(f'tmp/{img_name}_back.png', img)

extras/facexlib/utils/misc.py

@@ -1,118 +0,0 @@
-import cv2
-import os
-import os.path as osp
-import torch
-from torch.hub import download_url_to_file, get_dir
-from urllib.parse import urlparse
-
-ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-
-def imwrite(img, file_path, params=None, auto_mkdir=True):
-    """Write image to file.
-
-    Args:
-        img (ndarray): Image array to be written.
-        file_path (str): Image file path.
-        params (None or list): Same as opencv's :func:`imwrite` interface.
-        auto_mkdir (bool): If the parent folder of `file_path` does not exist,
-            whether to create it automatically.
-
-    Returns:
-        bool: Successful or not.
-    """
-    if auto_mkdir:
-        dir_name = os.path.abspath(os.path.dirname(file_path))
-        os.makedirs(dir_name, exist_ok=True)
-    return cv2.imwrite(file_path, img, params)
-
-
-def img2tensor(imgs, bgr2rgb=True, float32=True):
-    """Numpy array to tensor.
-
-    Args:
-        imgs (list[ndarray] | ndarray): Input images.
-        bgr2rgb (bool): Whether to change bgr to rgb.
-        float32 (bool): Whether to change to float32.
-
-    Returns:
-        list[tensor] | tensor: Tensor images. If returned results only have
-            one element, just return tensor.
-    """
-
-    def _totensor(img, bgr2rgb, float32):
-        if img.shape[2] == 3 and bgr2rgb:
-            if img.dtype == 'float64':
-                img = img.astype('float32')
-            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-        img = torch.from_numpy(img.transpose(2, 0, 1))
-        if float32:
-            img = img.float()
-        return img
-
-    if isinstance(imgs, list):
-        return [_totensor(img, bgr2rgb, float32) for img in imgs]
-    else:
-        return _totensor(imgs, bgr2rgb, float32)
-
-
-def load_file_from_url(url, model_dir=None, progress=True, file_name=None, save_dir=None):
-    """Ref:https://github.com/1adrianb/face-alignment/blob/master/face_alignment/utils.py
-    """
-    if model_dir is None:
-        hub_dir = get_dir()
-        model_dir = os.path.join(hub_dir, 'checkpoints')
-
-    if save_dir is None:
-        save_dir = os.path.join(ROOT_DIR, model_dir)
-    os.makedirs(save_dir, exist_ok=True)
-
-    parts = urlparse(url)
-    filename = os.path.basename(parts.path)
-    if file_name is not None:
-        filename = file_name
-    cached_file = os.path.abspath(os.path.join(save_dir, filename))
-    if not os.path.exists(cached_file):
-        print(f'Downloading: "{url}" to {cached_file}\n')
-        download_url_to_file(url, cached_file, hash_prefix=None, progress=progress)
-    return cached_file
-
-
-def scandir(dir_path, suffix=None, recursive=False, full_path=False):
-    """Scan a directory to find the interested files.
-    Args:
-        dir_path (str): Path of the directory.
-        suffix (str | tuple(str), optional): File suffix that we are
-            interested in. Default: None.
-        recursive (bool, optional): If set to True, recursively scan the
-            directory. Default: False.
-        full_path (bool, optional): If set to True, include the dir_path.
-            Default: False.
-    Returns:
-        A generator for all the interested files with relative paths.
-    """
-
-    if (suffix is not None) and not isinstance(suffix, (str, tuple)):
-        raise TypeError('"suffix" must be a string or tuple of strings')
-
-    root = dir_path
-
-    def _scandir(dir_path, suffix, recursive):
-        for entry in os.scandir(dir_path):
-            if not entry.name.startswith('.') and entry.is_file():
-                if full_path:
-                    return_path = entry.path
-                else:
-                    return_path = osp.relpath(entry.path, root)
-
-                if suffix is None:
-                    yield return_path
-                elif return_path.endswith(suffix):
-                    yield return_path
-            else:
-                if recursive:
-                    yield from _scandir(entry.path, suffix=suffix, recursive=recursive)
-                else:
-                    continue
-
-    return _scandir(dir_path, suffix=suffix, recursive=recursive)