fixed model loading

app.py

@@ -4,7 +4,8 @@ os.environ["CUDA_VISIBLE_DEVICES"] = ""
 import gradio as gr
 
 #from unsloth import FastLanguageModel
-from transformers import TextIteratorStreamer, AutoModelForCausalLM, AutoTokenizer
+from peft import AutoPeftModelForCausalLM
+from transformers import TextIteratorStreamer, AutoTokenizer
 from threading import Thread
 
 """
@@ -37,7 +38,7 @@ class MyModel:
             # )
             # FastLanguageModel.for_inference(client) # Enable native 2x faster inference
             tokenizer = AutoTokenizer.from_pretrained(model)
-            client = AutoModelForCausalLM.from_pretrained(model)
+            client = AutoPeftModelForCausalLM.from_pretrained(model, load_in_4bit=True)
 
             self.client = client
             self.tokenizer = tokenizer

requirements.txt

@@ -1,3 +1,4 @@
 huggingface_hub==0.25.2
 transformers>=4.45.1
-torch
+torch
+peft