import torch import torch.nn as nn from torch.utils.checkpoint import checkpoint from transformers import T5Tokenizer, T5EncoderModel, CLIPTokenizer, CLIPTextModel, AutoProcessor, CLIPVisionModel, CLIPImageProcessor import open_clip from ldm.util import default, count_params import kornia # import clip from einops import rearrange class AbstractEncoder(nn.Module): def __init__(self): super().__init__() def encode(self, *args, **kwargs): raise NotImplementedError class IdentityEncoder(AbstractEncoder): def encode(self, x): return x class ClassEmbedder(nn.Module): def __init__(self, embed_dim, n_classes=1000, key='class', ucg_rate=0.1): super().__init__() self.key = key self.embedding = nn.Embedding(n_classes, embed_dim) self.n_classes = n_classes self.ucg_rate = ucg_rate def forward(self, batch, key=None, disable_dropout=False): if key is None: key = self.key # this is for use in crossattn c = batch[key][:, None] if self.ucg_rate > 0. and not disable_dropout: mask = 1. - torch.bernoulli(torch.ones_like(c) * self.ucg_rate) c = mask * c + (1-mask) * torch.ones_like(c)*(self.n_classes-1) c = c.long() c = self.embedding(c) return c def get_unconditional_conditioning(self, bs, device="cuda"): uc_class = self.n_classes - 1 # 1000 classes --> 0 ... 999, one extra class for ucg (class 1000) uc = torch.ones((bs,), device=device) * uc_class uc = {self.key: uc} return uc def disabled_train(self, mode=True): """Overwrite model.train with this function to make sure train/eval mode does not change anymore.""" return self class FrozenT5Embedder(AbstractEncoder): """Uses the T5 transformer encoder for text""" def __init__(self, version="google/t5-v1_1-large", device="cuda", max_length=77, freeze=True): # others are google/t5-v1_1-xl and google/t5-v1_1-xxl super().__init__() self.tokenizer = T5Tokenizer.from_pretrained(version) self.transformer = T5EncoderModel.from_pretrained(version) self.device = device self.max_length = max_length # TODO: typical value? if freeze: self.freeze() def freeze(self): self.transformer = self.transformer.eval() #self.train = disabled_train for param in self.parameters(): param.requires_grad = False def forward(self, text): batch_encoding = self.tokenizer(text, truncation=True, max_length=self.max_length, return_length=True, return_overflowing_tokens=False, padding="max_length", return_tensors="pt") tokens = batch_encoding["input_ids"].to(self.device) outputs = self.transformer(input_ids=tokens) z = outputs.last_hidden_state return z def encode(self, text): return self(text) class FrozenCLIPEmbedder(AbstractEncoder): """Uses the CLIP transformer encoder for text (from huggingface)""" LAYERS = [ "last", "pooled", "hidden" ] def __init__(self, version="openai/clip-vit-large-patch14", device="cuda", max_length=77, freeze=True, layer="last", layer_idx=None): # clip-vit-base-patch32 super().__init__() assert layer in self.LAYERS self.tokenizer = CLIPTokenizer.from_pretrained(version) self.transformer = CLIPTextModel.from_pretrained(version) self.device = device self.max_length = max_length if freeze: self.freeze() self.layer = layer self.layer_idx = layer_idx if layer == "hidden": assert layer_idx is not None assert 0 <= abs(layer_idx) <= 12 def freeze(self): self.transformer = self.transformer.eval() #self.train = disabled_train for param in self.parameters(): param.requires_grad = False def forward(self, text): batch_encoding = self.tokenizer(text, truncation=True, max_length=self.max_length, return_length=True, return_overflowing_tokens=False, padding="max_length", return_tensors="pt") tokens = batch_encoding["input_ids"].to(self.device) outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden") if self.layer == "last": z = outputs.last_hidden_state elif self.layer == "pooled": z = outputs.pooler_output[:, None, :] else: z = outputs.hidden_states[self.layer_idx] # print(z.shape) return z def encode(self, text): return self(text) # class FrozenCLIPDualEmbedder(AbstractEncoder): # """Uses the CLIP transformer encoder for text (from huggingface)""" # LAYERS = [ # "last", # "pooled", # "hidden" # ] # def __init__(self, version="openai/clip-vit-large-patch14", device="cuda", max_length=77, # freeze=True, layer="last", layer_idx=None): # clip-vit-base-patch32 # super().__init__() # assert layer in self.LAYERS # self.tokenizer = CLIPTokenizer.from_pretrained(version) # self.transformer = CLIPTextModel.from_pretrained(version) # # self.processor = CLIPImageProcessor.from_pretrained(version) # # self.imagetransformer = CLIPVisionModel.from_pretrained(version) # self.ImageEmbedder=FrozenClipImageEmbedder() # self.device = device # self.max_length = max_length # if freeze: # self.freeze() # self.layer = layer # self.layer_idx = layer_idx # if layer == "hidden": # assert layer_idx is not None # assert 0 <= abs(layer_idx) <= 12 # def freeze(self): # self.transformer = self.transformer.eval() # #self.train = disabled_train # for name,param in self.named_parameters(): # if not "imagetransformer" in name and not "imageconv" in name and not "ImageEmbedder" in name: # # print(name,param) # param.requires_grad = False # else: # param.requires_grad = True # # print(name) # def forward(self, text): # # print("text:",len(text)) # # if len(text)==1: # # txt=text[0] # # hint_image=None # # elif len(text)==2: # # txt,hint_image=text # txt,hint_image=text # # print(hint_image.shape) # batch_encoding = self.tokenizer(txt, truncation=True, max_length=self.max_length, return_length=True, # return_overflowing_tokens=False, padding="max_length", return_tensors="pt") # tokens = batch_encoding["input_ids"].to(self.device) # outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden") # # input_image_batch_encoding = self.processor(input_image,return_tensors="pt") # # ii_tokens = input_image_batch_encoding["input_ids"].to(self.device) # # ii_outputs = self.imagetransformer(input_ids=ii_tokens, output_hidden_states=self.layer=="hidden") # # hint_image_batch_encoding = self.processor(hint_image,return_tensors="pt") # # hi_tokens = hint_image_batch_encoding["input_ids"].to(self.device) # # hi_outputs = self.imagetransformer(input_ids=hi_tokens, output_hidden_states=self.layer=="hidden") # # hint_outputs = hi_outputs-ii_outputs # # if hint_image==None: # # if self.layer == "last": # # z = outputs.last_hidden_state # # elif self.layer == "pooled": # # z = outputs.pooler_output[:, None, :] # # else: # # z = outputs.hidden_states[self.layer_idx] # # # print("z",z.shape) # # return z # hint_outputs=self.ImageEmbedder(hint_image) # # print("hint_outputs",hint_outputs.shape) # # print("prompt",outputs.last_hidden_state.shape) # if self.layer == "last": # z = torch.cat((outputs.last_hidden_state,hint_outputs.unsqueeze(0)),1)#torch.cat((outputs.last_hidden_state,hint_outputs.last_hidden_state),1)#torch.cat((outputs.last_hidden_state,hint_outputs.unsqueeze(0)),1) # elif self.layer == "pooled": # z = torch.cat((outputs.pooler_output[:, None, :],hint_outputs.unsqueeze(0)),1) # else: # z = torch.cat((outputs.hidden_states[self.layer_idx],hint_outputs.unsqueeze(0)),1) # # print("z",z.shape) # return z # def encode(self, text): # # print(text.shape) # return self(text) class FrozenCLIPDualEmbedder(AbstractEncoder): """Uses the CLIP transformer encoder for text (from huggingface)""" LAYERS = [ "last", "pooled", "hidden" ] def __init__(self, version="openai/clip-vit-large-patch14", device="cuda", max_length=77, freeze=True, layer="last", layer_idx=None): # clip-vit-base-patch32 super().__init__() assert layer in self.LAYERS self.tokenizer = CLIPTokenizer.from_pretrained(version) self.transformer = CLIPTextModel.from_pretrained(version) # self.processor = CLIPImageProcessor.from_pretrained(version) # self.imagetransformer = CLIPVisionModel.from_pretrained(version) self.ImageEmbedder=FrozenClipImageEmbedder() self.device = device self.max_length = max_length if freeze: self.freeze() self.layer = layer self.layer_idx = layer_idx if layer == "hidden": assert layer_idx is not None assert 0 <= abs(layer_idx) <= 12 print("pooled") def freeze(self): # self.transformer = self.transformer.eval() #self.train = disabled_train for name,param in self.named_parameters(): # print(name) # if not "imagetransformer" in name and not "imageconv" in name and not "ImageEmbedder" in name: param.requires_grad = False # if not "ImageEmbedder" in name: # # print(name,param) # param.requires_grad = False # else: # param.requires_grad = True def forward(self, text): # pdb.set_trace() # print("text:",len(text)) # if len(text)==1: # txt=text[0] # hint_image=None # elif len(text)==2: # txt,hint_image=text txt,hint_image=text # if hint_image==None: # batch_encoding = self.tokenizer(txt, truncation=True, max_length=self.max_length, return_length=True, # return_overflowing_tokens=False, padding="max_length", return_tensors="pt") # tokens = batch_encoding["input_ids"].to(self.device) # outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden") # prompt_outputs=outputs.last_hidden_state # return prompt_outputs # else: # hint_image.requires_grad_(True) # print(hint_image.shape) batch_encoding = self.tokenizer(txt, truncation=True, max_length=self.max_length, return_length=True, return_overflowing_tokens=False, padding="max_length", return_tensors="pt") tokens = batch_encoding["input_ids"].to(self.device) outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden") prompt_outputs=outputs.last_hidden_state # prompt_outputs=outputs.last_hidden_state.detach().requires_grad_(True) # prompt_outputs.retain_grad() # input_image_batch_encoding = self.processor(input_image,return_tensors="pt") # ii_tokens = input_image_batch_encoding["input_ids"].to(self.device) # ii_outputs = self.imagetransformer(input_ids=ii_tokens, output_hidden_states=self.layer=="hidden") # hint_image_batch_encoding = self.processor(hint_image,return_tensors="pt") # hi_tokens = hint_image_batch_encoding["input_ids"].to(self.device) # hi_outputs = self.imagetransformer(input_ids=hi_tokens, output_hidden_states=self.layer=="hidden") # hint_outputs = hi_outputs-ii_outputs # if hint_image==None: # if self.layer == "last": # z = outputs.last_hidden_state # elif self.layer == "pooled": # z = outputs.pooler_output[:, None, :] # else: # z = outputs.hidden_states[self.layer_idx] # # print("z",z.shape) # return z # pdb.set_trace() outputs = self.ImageEmbedder(hint_image) # image_embeds = outputs.pooler_output #outputs.image_embeds image_embeds = outputs.pooler_output # print(image_embeds.shape) # last_hidden_state = outputs.last_hidden_state # pooled_output = outputs.pooler_output # print("hint_outputs",last_hidden_state.shape) # print("pooled_output", pooled_output.shape) # print("prompt",prompt_outputs.shape) if self.layer == "last": # print(prompt_outputs.shape) # print(image_embeds.shape) z = torch.cat((prompt_outputs,image_embeds.unsqueeze(1)),1)#,hint_outputs.unsqueeze(0)),1) # z = torch.cat((prompt_outputs,hint_outputs.last_hidden_state),1)#,hint_outputs.unsqueeze(0)),1) elif self.layer == "pooled": z = torch.cat((outputs.pooler_output[:, None, :],hint_outputs.unsqueeze(0)),1) else: z = torch.cat((outputs.hidden_states[self.layer_idx],hint_outputs.unsqueeze(0)),1) return z # def __init__(self, version="openai/clip-vit-large-patch14", device="cuda", max_length=77, # freeze=True, layer="last", layer_idx=None): # clip-vit-base-patch32 # super().__init__() # assert layer in self.LAYERS # # self.processor = CLIPImageProcessor.from_pretrained(version) # # self.imagetransformer = CLIPVisionModel.from_pretrained(version) # self.ImageEmbedder=FrozenClipImageEmbedder() # self.device = device # self.max_length = max_length # if freeze: # self.freeze() # self.layer = layer # self.layer_idx = layer_idx # if layer == "hidden": # assert layer_idx is not None # assert 0 <= abs(layer_idx) <= 12 # def freeze(self): # #self.train = disabled_train # for name,param in self.named_parameters(): # if not "imagetransformer" in name and not "imageconv" in name and not "ImageEmbedder" in name: # # print(name,param) # param.requires_grad = False # else: # param.requires_grad = True # # print(name) # def forward(self, txt,hint_image): # # pdb.set_trace() # hint_outputs=self.ImageEmbedder(hint_image) # # print("hint_outputs",hint_outputs.shape) # # print("prompt",outputs.last_hidden_state.shape) # if self.layer == "last": # print(txt.shape) # print(hint_outputs.last_hidden_state.shape) # z = torch.cat((txt,hint_outputs.last_hidden_state),1)#,hint_outputs.unsqueeze(0)),1) # elif self.layer == "pooled": # z = torch.cat((txt,hint_outputs.unsqueeze(0)),1) # else: # z = torch.cat((txt,hint_outputs.unsqueeze(0)),1) # # print("z",z.shape) # return z def encode(self, text): # if isinstance(text, dict): # txt,hint_image=text['c_crossattn'][0] # txt=txt # else: # txt,hint_image=text # txt = text txt, x = text # if x==None: # return self((txt,x)) # print(x.shape) if len(x.shape) == 3: x = x[..., None] x = rearrange(x, 'b h w c -> b c h w') x = x.to(memory_format=torch.contiguous_format).float() x = [x[i] for i in range(x.shape[0])] return self((txt, x)) class FrozenOpenCLIPEmbedder(AbstractEncoder): """ Uses the OpenCLIP transformer encoder for text """ LAYERS = [ #"pooled", "last", "penultimate" ] def __init__(self, arch="ViT-H-14", version="laion2b_s32b_b79k", device="cuda", max_length=77, freeze=True, layer="last"): super().__init__() assert layer in self.LAYERS model, _, _ = open_clip.create_model_and_transforms(arch, device=torch.device('cpu'), pretrained=version) del model.visual self.model = model self.device = device self.max_length = max_length if freeze: self.freeze() self.layer = layer if self.layer == "last": self.layer_idx = 0 elif self.layer == "penultimate": self.layer_idx = 1 else: raise NotImplementedError() def freeze(self): self.model = self.model.eval() for param in self.parameters(): param.requires_grad = False def forward(self, text): tokens = open_clip.tokenize(text) z = self.encode_with_transformer(tokens.to(self.device)) return z def encode_with_transformer(self, text): x = self.model.token_embedding(text) # [batch_size, n_ctx, d_model] x = x + self.model.positional_embedding x = x.permute(1, 0, 2) # NLD -> LND x = self.text_transformer_forward(x, attn_mask=self.model.attn_mask) x = x.permute(1, 0, 2) # LND -> NLD x = self.model.ln_final(x) return x def text_transformer_forward(self, x: torch.Tensor, attn_mask = None): for i, r in enumerate(self.model.transformer.resblocks): if i == len(self.model.transformer.resblocks) - self.layer_idx: break if self.model.transformer.grad_checkpointing and not torch.jit.is_scripting(): x = checkpoint(r, x, attn_mask) else: x = r(x, attn_mask=attn_mask) return x def encode(self, text): return self(text) class FrozenCLIPT5Encoder(AbstractEncoder): def __init__(self, clip_version="openai/clip-vit-large-patch14", t5_version="google/t5-v1_1-xl", device="cuda", clip_max_length=77, t5_max_length=77): super().__init__() self.clip_encoder = FrozenCLIPEmbedder(clip_version, device, max_length=clip_max_length) self.t5_encoder = FrozenT5Embedder(t5_version, device, max_length=t5_max_length) print(f"{self.clip_encoder.__class__.__name__} has {count_params(self.clip_encoder)*1.e-6:.2f} M parameters, " f"{self.t5_encoder.__class__.__name__} comes with {count_params(self.t5_encoder)*1.e-6:.2f} M params.") def encode(self, text): return self(text) def forward(self, text): clip_z = self.clip_encoder.encode(text) t5_z = self.t5_encoder.encode(text) return [clip_z, t5_z] class FrozenClipImageEmbedder(nn.Module): """ Uses the CLIP image encoder. """ def __init__( self, model='ViT-B/16', #ViT-L/14 jit=False, device='cuda' if torch.cuda.is_available() else 'cpu', antialias=False, ): super().__init__() # self.model, _ = clip.load(name=model, device=device, jit=jit) # self.model.requires_grad_(True) self.imageconv = nn.Conv2d(4,3,(3,3),padding=1)#.cuda() self.antialias = antialias self.register_buffer('mean', torch.Tensor([0.48145466, 0.4578275, 0.40821073]), persistent=False) self.register_buffer('std', torch.Tensor([0.26862954, 0.26130258, 0.27577711]), persistent=False) self.device = device self.processor = CLIPImageProcessor.from_pretrained("openai/clip-vit-base-patch32") self.model = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch32") # self.imagetransformer = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch16") # def preprocess(self, x): # # normalize to [0,1] # # print(x.shape) # # pdb.set_trace() # x = kornia.geometry.resize(x, (224, 224), # interpolation='bicubic',align_corners=True, # antialias=self.antialias) # # print("after",x.shape) # # x = (x + 1.) / 2. # print(x) # # renormalize according to clip # x = kornia.enhance.normalize(x, self.mean, self.std) # # print("after1111111",x.shape) # return x def forward(self, x): # x is assumed to be in range [-1,1] # pdb.set_trace() # with torch.set_grad_enabled(True): # print("before",x.shape) # x=self.imageconv(x) # print("after",x.shape) # x = x.tolist() x = self.processor(x, return_tensors="pt") # print(x) # pdb.set_trace() x['pixel_values'] = x['pixel_values'].to(self.device) outputs = self.model(**x) return outputs # class FrozenClipImageEmbedder(nn.Module): # """ # Uses the CLIP image encoder. # """ # def __init__( # self, # model='ViT-B/16', # jit=False, # device='cuda' if torch.cuda.is_available() else 'cpu', # antialias=False, # ): # super().__init__() # self.model, _ = clip.load(name=model, device=device, jit=jit) # # self.imageconv = nn.Conv2d(4,3,(3,3),stride=2) # self.antialias = antialias # self.register_buffer('mean', torch.Tensor([0.48145466, 0.4578275, 0.40821073]), persistent=False) # self.register_buffer('std', torch.Tensor([0.26862954, 0.26130258, 0.27577711]), persistent=False) # def preprocess(self, x): # # normalize to [0,1] # # print(x.shape) # x = kornia.geometry.resize(x, (224, 224), # interpolation='bicubic',align_corners=True, # antialias=self.antialias) # # print("after",x.shape) # x = (x + 1.) / 2. # # renormalize according to clip # x = kornia.enhance.normalize(x, self.mean, self.std) # # print("after1111111",x.shape) # return x # def forward(self, x): # # x is assumed to be in range [-1,1] # # x=self.imageconv(x) # return self.model.encode_image(self.preprocess(x)) # class FrozenClipImageEmbedder(nn.Module): # """ # Uses the CLIP image encoder. # """ # def __init__( # self, # model='ViT-B/16', #ViT-L/14 # jit=False, # device='cuda' if torch.cuda.is_available() else 'cpu', # antialias=False, # ): # super().__init__() # self.model, _ = clip.load(name=model, device=device, jit=jit) # # self.model.requires_grad_(True) # # self.imageconv = nn.Conv2d(4,3,(3,3),padding=1)#.cuda()#padding=1 #stride=2 # self.antialias = antialias # self.register_buffer('mean', torch.Tensor([0.48145466, 0.4578275, 0.40821073]), persistent=False) # self.register_buffer('std', torch.Tensor([0.26862954, 0.26130258, 0.27577711]), persistent=False) # # self.processor = CLIPImageProcessor.from_pretrained("openai/clip-vit-large-patch14") # self.imagetransformer = CLIPVisionModel.from_pretrained("openai/clip-vit-base-patch16") # def preprocess(self, x): # # normalize to [0,1] # # print(x.shape) # # pdb.set_trace() # x = kornia.geometry.resize(x, (224, 224), # interpolation='bicubic',align_corners=True, # antialias=self.antialias) # # print("after",x.shape) # x = (x + 1.) / 2. # # renormalize according to clip # x = kornia.enhance.normalize(x, self.mean, self.std) # # print("after1111111",x.shape) # return x # def forward(self, x): # # x is assumed to be in range [-1,1] # # x=self.imageconv(x) # return self.imagetransformer(self.preprocess(x), output_hidden_states="last"=="hidden") #self.model.encode_image(self.preprocess(x))