handler.py

from typing import Dict, List, Any
from transformers import AutoTokenizer, AutoConfig, AutoModelForSequenceClassification
import torch



class EndpointHandler:
    def __init__(self, path=""):
        # load model and processor from path
        guider_config = AutoConfig.from_pretrained(path)
        self.model =  AutoModelForSequenceClassification.from_pretrained(path, config=guider_config)
        self.tokenizer = AutoTokenizer.from_pretrained(path)

    def __call__(self, data: Dict[str, Any]) -> Dict[str, Any]:
        """
        Args:
            data (:dict:):
                The payload with the text prompt.
        """
        # process input
        gen_outputs_no_input_decoded = data.pop("gen_outputs_no_input_decoded", data)

        # Guiding the model with his ranking,
        guider_inputs = self.tokenizer([gen_output_no_input_decoded for gen_output_no_input_decoded in gen_outputs_no_input_decoded],
                                        return_tensors='pt', padding=True, truncation=True)

        guider_outputs = self.model(**guider_inputs)

        # the slicing at the end [:,x]: x=0 for negative, x=1 for neutral, x=2 for positive
        guider_predictions = torch.nn.functional.softmax(guider_outputs.logits, dim=-1)[:, 0].tolist()
        return {"guider_predictions": guider_predictions}