| from transformers import AutoTokenizer, AutoModelForCausalLM, LlamaConfig | |
| from config.config import settings | |
| from sentence_transformers import SentenceTransformer | |
| import torch | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| class ModelService: | |
| _instance = None | |
| def __new__(cls): | |
| if cls._instance is None: | |
| cls._instance = super().__new__(cls) | |
| cls._instance._initialized = False | |
| return cls._instance | |
| def __init__(self): | |
| if not self._initialized: | |
| self._initialized = True | |
| self._load_models() | |
| def _load_models(self): | |
| try: | |
| # Load tokenizer | |
| #self.tokenizer = AutoTokenizer.from_pretrained(settings.MODEL_NAME) | |
| ## Load model configuration | |
| #config = LlamaConfig.from_pretrained(settings.MODEL_NAME) | |
| ## Check quantization type and adjust accordingly | |
| #if config.get('quantization_config', {}).get('type', '') == 'compressed-tensors': | |
| # logger.warning("Quantization type 'compressed-tensors' is not supported. Switching to 'bitsandbytes_8bit'.") | |
| # config.quantization_config['type'] = 'bitsandbytes_8bit' | |
| ## Load model with the updated configuration | |
| #self.model = AutoModelForCausalLM.from_pretrained( | |
| # settings.MODEL_NAME, | |
| # config=config, | |
| # torch_dtype=torch.float16 if settings.DEVICE == "cuda" else torch.float32, | |
| # device_map="auto" if settings.DEVICE == "cuda" else None | |
| #) | |
| #----- | |
| # Load Llama 3.2 model | |
| model_name = settings.MODEL_NAME #"meta-llama/Llama-3.2-3B-Instruct" # Replace with the exact model path | |
| tokenizer = AutoTokenizer.from_pretrained(model_name) | |
| #model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.float16) | |
| self.model = AutoModelForCausalLM.from_pretrained(model_name, device_map=None, torch_dtype=torch.float32) | |
| # Load sentence embedder | |
| self.embedder = SentenceTransformer(settings.EMBEDDER_MODEL) | |
| except Exception as e: | |
| logger.error(f"Error loading models: {e}") | |
| raise | |
| def get_models(self): | |
| return self.tokenizer, self.model, self.embedder |