Create app.py

app.py

@@ -0,0 +1,65 @@
+import torch
+import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel, PeftConfig
+
+# Load model and tokenizer
+MODEL_PATH = "sagar007/phi2_finetune"
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
+tokenizer.pad_token = tokenizer.eos_token
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16,
+    bnb_4bit_use_double_quant=False
+)
+
+base_model = AutoModelForCausalLM.from_pretrained(
+    "microsoft/phi-2",
+    quantization_config=bnb_config,
+    device_map="auto",
+    trust_remote_code=True
+)
+
+peft_config = PeftConfig.from_pretrained(MODEL_PATH)
+model = PeftModel.from_pretrained(base_model, MODEL_PATH)
+model.eval()
+
+def generate_response(instruction, max_length=512):
+    prompt = f"Instruction: {instruction}\nResponse:"
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs,
+            max_length=max_length,
+            num_return_sequences=1,
+            temperature=0.7,
+            top_p=0.9,
+            do_sample=True
+        )
+    
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return response.split("Response:")[1].strip()
+
+def chatbot(message, history):
+    response = generate_response(message)
+    return response
+
+demo = gr.ChatInterface(
+    chatbot,
+    title="Fine-tuned Phi-2 Chatbot",
+    description="This is a chatbot using a fine-tuned version of the Phi-2 model.",
+    theme="default",
+    examples=[
+        "Explain the concept of machine learning.",
+        "Write a short story about a robot learning to paint.",
+        "What are some effective ways to reduce stress?",
+    ],
+    cache_examples=False,
+)
+
+if __name__ == "__main__":
+    demo.launch()