update/tried 8bit cpu quant route

app.py

@@ -7,7 +7,11 @@ def load_model():
     base_model_name = "unsloth/llama-3.2-1b-instruct-bnb-4bit"  # Replace with your base model name
     lora_model_name = "sreyanghosh/lora_model"  # Replace with your LoRA model path
     tokenizer = AutoTokenizer.from_pretrained(base_model_name)
-    model = AutoModelForCausalLM.from_pretrained(base_model_name, device_map="auto")
+    model = AutoModelForCausalLM.from_pretrained(
+        base_model_name, 
+        device_map="auto" if torch.cuda.is_available() else None,
+        load_in_8bit=not torch.cuda.is_available(),
+    )
     model = PeftModel.from_pretrained(model, lora_model_name)
     model.eval()
     return tokenizer, model

requirements.txt

@@ -3,5 +3,4 @@ gradio
 transformers
 peft
 torch
-# unsloth
-bitsandbytes
+bitsandbytes>=0.37