Update app.py

app.py

@@ -1,6 +1,35 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
 import gradio as gr
+import torch
+from peft import PeftConfig, PeftModel
 
-gr.load("models/TurtleLiu/mistral7b_psychology_bot").launch()
+# Loading PEFT model
+PEFT_MODEL = "TurtleLiu/mistral7b_psychology_bot"
+
+config = PeftConfig.from_pretrained(PEFT_MODEL)
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit= True,
+    bnb_4bit_quant_type= "nf4",
+    bnb_4bit_compute_dtype= torch.bfloat16,
+    bnb_4bit_use_double_quant= False,
+)
+
+peft_base_model = AutoModelForCausalLM.from_pretrained(
+    config.base_model_name_or_path,
+    return_dict=True,
+    quantization_config=bnb_config,
+    device_map="auto",
+    trust_remote_code=True,
+)
+
+model = PeftModel.from_pretrained(peft_base_model, PEFT_MODEL)
+model = model.merge_and_unload()
+
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path, trust_remote_code=True)
+tokenizer.pad_token = tokenizer.eos_token
+tokenizer.padding_side = "right"
 
 '''
 from huggingface_hub import InferenceClient