off_topic.py

import re
import time
import asyncio
from io import BytesIO
from typing import List, Optional

import httpx
import matplotlib.pyplot as plt
import numpy as np
import torch
import PIL
from transformers import CLIPModel, CLIPProcessor
from PIL import Image


class OffTopicDetector:
    def __init__(self, model_id: str, device: Optional[str] = None):
        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
        self.processor = CLIPProcessor.from_pretrained(model_id)
        self.model = CLIPModel.from_pretrained(model_id).to(self.device)

    def predict_probas(self, images: List[PIL.Image.Image], domain: str,
                valid_templates: Optional[List[str]] = None,
                invalid_classes: Optional[List[str]] = None,
                autocast: bool = True):
        if valid_templates:
            valid_classes = [template.format(domain) for template in valid_templates]
        else:
            valid_classes = [f"a photo of {domain}", f"brochure with {domain} image", f"instructions for {domain}", f"{domain} diagram"]
        if not invalid_classes:
            invalid_classes = ["promotional ad with store information", "promotional text", "google maps screenshot", "business card", "qr code"]
        n_valid = len(valid_classes)
        classes = valid_classes + invalid_classes
        print(f"Valid classes: {valid_classes}", f"Invalid classes: {invalid_classes}", sep="\n")
        n_classes = len(classes)

        start = time.time()
        inputs = self.processor(text=classes, images=images, return_tensors="pt", padding=True).to(self.device)
        if self.device == "cpu" and autocast is True:
            print("Disabling autocast due to device='cpu'.")
            autocast = False
        with torch.autocast(self.device, enabled=autocast):
            with torch.no_grad():
                outputs = self.model(**inputs)
                probas = outputs.logits_per_image.softmax(dim=1).cpu().numpy() # we can take the softmax to get the label probabilities
        end = time.time()
        duration = end - start
        print(f"Device: {self.device}",
              f"Response time: {duration}s",
              f"Response time per image: {round(duration/len(images), 2) * 1000}ms",
              sep="\n")
        valid_probas = probas[:, 0:n_valid].sum(axis=1, keepdims=True)
        invalid_probas = probas[:, n_valid:n_classes].sum(axis=1, keepdims=True)
        return probas, valid_probas, invalid_probas

    def show(self, images: List[PIL.Image.Image], valid_probas: np.ndarray, n_cols: int = 3, title: Optional[str] = None, threshold: Optional[float] = None):
        if threshold is not None:
            prediction = self.apply_threshold(valid_probas, threshold)
            title_scores = [f"Valid: {pred.squeeze()}" for pred in prediction]
        else:
            prediction = np.round(valid_probas[:, 0], 2)
            title_scores = [f"Valid: {pred:.2f}" for pred in prediction]
        n_images = len(images)
        n_rows = int(np.ceil(n_images / n_cols))
        fig, axes = plt.subplots(n_rows, n_cols, figsize=(16, 16))
        for i, ax in enumerate(axes.ravel()):
            ax.axis("off")
            try:
                ax.imshow(images[i])
                ax.set_title(title_scores[i])
            except IndexError:
                continue
        if title:
            fig.suptitle(title)
        fig.tight_layout()
        return

    def predict_item_probas(self, url_or_id: str,
                valid_templates: Optional[List[str]] = None,
                invalid_classes: Optional[List[str]] = None):
        images, domain = self.get_item_data(url_or_id)
        probas, valid_probas, invalid_probas = self.predict_probas(images, domain, valid_templates,
                                                            invalid_classes)
        return images, domain, probas, valid_probas, invalid_probas

    def apply_threshold(self, valid_probas: np.ndarray, threshold: float = 0.4):
        return valid_probas >= threshold

    def get_item_data(self, url_or_id: str):
        if url_or_id.startswith("http"):
            item_id = "".join(url_or_id.split("/")[3].split("-")[:2])
        else:
            item_id = re.sub("-", "", url_or_id)
        response = httpx.get(f"https://api.mercadolibre.com/items/{item_id}").json()
        domain = re.sub("_", " ", response["domain_id"].split("-")[-1]).lower()
        img_urls = [x["url"] for x in response["pictures"]]
        images = self.get_images(img_urls)
        return images, domain

    def get_images(self, urls: List[str]):
        start = time.time()
        images = asyncio.run(self._gather_download_tasks(urls))
        end = time.time()
        duration = end - start
        print(f"Download time: {duration}s",
              f"Download time per image: {round(duration/len(urls), 2) * 1000}ms",
              sep="\n")
        return asyncio.run(self._gather_download_tasks(urls))

    async def _gather_download_tasks(self, urls: List[str]):

        async def _process_download(url: str, client: httpx.AsyncClient):
            response = await client.get(url)
            return Image.open(BytesIO(response.content))

        async with httpx.AsyncClient() as client:
            tasks = [_process_download(url, client) for url in urls]
            return await asyncio.gather(*tasks)

    @staticmethod
    def _non_async_get_item_data(url_or_id: str, save_images: bool = False):
        if url_or_id.startswith("http"):
            item_id = "".join(url_or_id.split("/")[3].split("-")[:2])
        else:
            item_id = re.sub("-", "", url_or_id)
        response = httpx.get(f"https://api.mercadolibre.com/items/{item_id}").json()
        domain = re.sub("_", " ", response["domain_id"].split("-")[-1]).lower()
        img_urls = [x["url"] for x in response["pictures"]]
        images = []
        for img_url in img_urls:
            img = httpx.get(img_url)
            images.append(Image.open(BytesIO(img.content)))
            if save_images:
                with open(re.sub("D_NQ_NP_", "", img_url.split("/")[-1]) , "wb") as f:
                    f.write(img.content)
        return images, domain