Spaces:
Runtime error
Runtime error
File size: 2,660 Bytes
413d4d0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
import numpy as np
from PIL import Image
import torch.nn.functional as F
from typing import List
from transformers import CLIPProcessor, CLIPModel
ROUND_DIGIT=3
NUM_ASPECT=5
CLIP_POINT_HIGH=0.97
CLIP_POINT_MID=0.9
CLIP_POINT_LOW=0.8
class MetricCLIP_sim():
def __init__(self, device = "cuda") -> None:
"""
Initialize a class MetricCLIP_sim with the specified device for testing temporal consistency of a given video.
Args:
device (str, optional): The device on which the model will run. Defaults to "cuda".
"""
self.device = device
self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
self.model.to(self.device)
self.tokenizer = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
def evaluate(self,frame_list:List[Image.Image]):
"""
Calculate the cosine similarity between the CLIP features of adjacent frames of a given video to test temporal consistency,
then quantize the orginal output based on some predefined thresholds.
Args:
frame_list:List[Image.Image], frames of the video used in calculation.
Returns:
clip_frame_score: float, the computed CLIP feature cosine similarity between each adjacent pair of frames and then averaged among all the pairs.
quantized_ans: int, the quantized value of the above avg CLIP-Sim scores based on pre-defined thresholds.
"""
device=self.model.device
frame_sim_list=[]
for f_idx in range(len(frame_list)-1):
frame_1 = frame_list[f_idx]
frame_2 = frame_list[f_idx+1]
input_1 = self.tokenizer(images=frame_1, return_tensors="pt", padding=True).to(device)
input_2 = self.tokenizer(images=frame_2, return_tensors="pt", padding=True).to(device)
output_1 = self.model.get_image_features(**input_1).flatten()
output_2 = self.model.get_image_features(**input_2).flatten()
cos_sim = F.cosine_similarity(output_1, output_2, dim=0).item()
frame_sim_list.append(cos_sim)
clip_frame_score = np.mean(frame_sim_list)
quantized_ans=0
if clip_frame_score >= CLIP_POINT_HIGH:
quantized_ans=4
elif clip_frame_score < CLIP_POINT_HIGH and clip_frame_score >= CLIP_POINT_MID:
quantized_ans=3
elif clip_frame_score < CLIP_POINT_MID and clip_frame_score >= CLIP_POINT_LOW:
quantized_ans=2
else:
quantized_ans=1
return clip_frame_score, quantized_ans
|