Update services/model_service.py

services/model_service.py

@@ -1,8 +1,8 @@
-from transformers import AutoTokenizer, AutoModelForCausalLM, LlamaConfig
-from config.config import settings
+from transformers import AutoTokenizer, AutoModelForCausalLM
 from sentence_transformers import SentenceTransformer
 import torch
 import logging
+from config.config import settings
 
 logger = logging.getLogger(__name__)
 
@@ -18,43 +18,44 @@ class ModelService:
     def __init__(self):
         if not self._initialized:
             self._initialized = True
+            self.tokenizer = None
+            self.model = None
+            self.embedder = None
             self._load_models()
 
     def _load_models(self):
         try:
+            logger.info("Loading models...")
+
             # Load tokenizer
-            #self.tokenizer = AutoTokenizer.from_pretrained(settings.MODEL_NAME)
-
-            ## Load model configuration
-            #config = LlamaConfig.from_pretrained(settings.MODEL_NAME)
-
-            ## Check quantization type and adjust accordingly
-            #if config.get('quantization_config', {}).get('type', '') == 'compressed-tensors':
-            #    logger.warning("Quantization type 'compressed-tensors' is not supported. Switching to 'bitsandbytes_8bit'.")
-            #    config.quantization_config['type'] = 'bitsandbytes_8bit'
-
-            ## Load model with the updated configuration
-            #self.model = AutoModelForCausalLM.from_pretrained(
-            #    settings.MODEL_NAME, 
-            #    config=config, 
-            #    torch_dtype=torch.float16 if settings.DEVICE == "cuda" else torch.float32,
-            #    device_map="auto" if settings.DEVICE == "cuda" else None
-            #)
-
-#-----
-            # Load Llama 3.2 model
-            model_name = settings.MODEL_NAME #"meta-llama/Llama-3.2-3B-Instruct"  # Replace with the exact model path
-            tokenizer = AutoTokenizer.from_pretrained(model_name)
-            #model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.float16)
-            self.model = AutoModelForCausalLM.from_pretrained(model_name, device_map=None, torch_dtype=torch.float32)
-
-            
+            self.tokenizer = AutoTokenizer.from_pretrained(settings.MODEL_NAME)
+            logger.info(f"Tokenizer for {settings.MODEL_NAME} loaded successfully.")
+
+            # Load language model
+            quantization_device = settings.DEVICE
+            quantization_bits = settings.QUANTIZATION_BITS
+
+            self.model = AutoModelForCausalLM.from_pretrained(
+                settings.MODEL_NAME,
+                torch_dtype=torch.float16 if quantization_device == "cuda" else torch.float32,
+                device_map="auto" if quantization_device == "cuda" else None,
+                load_in_8bit=(quantization_bits == 8),
+                trust_remote_code=True
+            )
+            logger.info(f"Model {settings.MODEL_NAME} loaded successfully on {quantization_device}.")
+
             # Load sentence embedder
             self.embedder = SentenceTransformer(settings.EMBEDDER_MODEL)
+            logger.info(f"Embedder {settings.EMBEDDER_MODEL} loaded successfully.")
 
         except Exception as e:
             logger.error(f"Error loading models: {e}")
-            raise
+            raise RuntimeError(f"Failed to initialize ModelService: {str(e)}")
 
     def get_models(self):
-        return self.tokenizer, self.model, self.embedder 
+        """
+        Returns the tokenizer, language model, and sentence embedder instances.
+        """
+        if not self.tokenizer or not self.model or not self.embedder:
+            raise RuntimeError("Models are not fully loaded.")
+        return self.tokenizer, self.model, self.embedder