Update app.py

app.py

@@ -1,42 +1,59 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
 
-# Initialize the client
-client = InferenceClient("davnas/Italian_Cousine_2.1")
+# Load model and tokenizer
+model_name = "davnas/Italian_Cousine_2.1"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    torch_dtype=torch.float32,  # Use float32 for CPU
+    low_cpu_mem_usage=True,
+    device_map="auto"
+)
 
-def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p):
-    # Format the prompt including history and system message
-    prompt = ""
-    
-    # Add system message if provided
-    if system_message:
-        prompt = f"{system_message}\n"
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    # Format the conversation
+    messages = [{"role": "system", "content": system_message}]
     
-    # Add conversation history
+    # Add history
     for user_msg, assistant_msg in history:
-        prompt += f"User: {user_msg}\nAssistant: {assistant_msg}\n"
+        messages.append({"role": "user", "content": user_msg})
+        messages.append({"role": "assistant", "content": assistant_msg})
     
     # Add current message
-    prompt += f"User: {message}\nAssistant:"
+    messages.append({"role": "user", "content": message})
+    
+    # Create the prompt using the tokenizer's chat template
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    )
+    
+    # Generate response
+    with torch.no_grad():
+        output_ids = model.generate(
+            input_ids,
+            max_new_tokens=max_tokens,
+            do_sample=True,
+            temperature=temperature,
+            top_p=top_p,
+            pad_token_id=tokenizer.pad_token_id,
+            streaming=True
+        )
     
-    response = ""
-    # Stream the response
-    for token in client.text_generation(
-        prompt,
-        max_new_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        response += token
-        yield response
+    # Decode and return the response
+    response = tokenizer.decode(output_ids[0][len(input_ids[0]):], skip_special_tokens=True)
+    return response
 
 # Create the interface
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
         gr.Textbox(
-            value="You are a friendly Chatbot.",
+            value="You are a professional chef assistant who provides accurate and detailed recipes.",
             label="System message"
         ),
         gr.Slider(
@@ -60,8 +77,10 @@ demo = gr.ChatInterface(
             step=0.05,
             label="Top-p (nucleus sampling)"
         ),
-    ]
+    ],
+    title="Italian Cuisine Chatbot",
+    description="Ask me anything about Italian cuisine or cooking!"
 )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(server_name="0.0.0.0", server_port=7860)