utils.py

import os
from transformers import AutoTokenizer, AutoModelForSequenceClassification

"""
from interfaces.manifesto import languages as languages_manifesto
from interfaces.manifesto import languages as languages_manifesto
from interfaces.manifesto import languages as languages_manifesto
"""

from interfaces.cap import build_huggingface_path as hf_cap_path
from interfaces.manifesto import build_huggingface_path as hf_manifesto_path
from interfaces.sentiment import build_huggingface_path as hf_sentiment_path
from interfaces.emotion import build_huggingface_path as hf_emotion_path


HF_TOKEN = os.environ["hf_read"]

models = [hf_manifesto_path(""), hf_sentiment_path(""), hf_emotion_path("")]
tokenizers = ["xlm-roberta-large"]

def download_hf_models():
    for model_id in models:
        model = AutoModelForSequenceClassification.from_pretrained(model_id, low_cpu_mem_usage=True, device_map="auto",
                                                                   token=HF_TOKEN)
        del model
    for tokenizer_id in tokenizers:
        tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
        del tokenizer