Update app.py

app.py

@@ -23,13 +23,12 @@ logger = logging.getLogger(__name__)
 load_dotenv()
 login(token=os.getenv("HUGGINGFACE_API_TOKEN"))
 
-# Initialize model with optimized settings
+# Initialize model with CPU-compatible settings
 model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 model = AutoModelForCausalLM.from_pretrained(
     model_name,
-    torch_dtype=torch.float16,
     device_map="auto",
-    load_in_8bit=True
+    torch_dtype=torch.float32,  # Use float32 for CPU compatibility
 )
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
@@ -42,7 +41,7 @@ class LegalTextSearchBot:
             )
             self.collection = self.astra_db.collection("legal_content")
             
-            # Initialize pipeline with optimized settings
+            # Initialize pipeline with CPU settings
             pipe = pipeline(
                 "text-generation",
                 model=model,
@@ -51,7 +50,6 @@ class LegalTextSearchBot:
                 temperature=0.7,
                 top_p=0.95,
                 repetition_penalty=1.15,
-                torch_dtype=torch.float16,
                 device_map="auto"
             )
             self.llm = HuggingFacePipeline(pipeline=pipe)