Update app.py

app.py

@@ -5,6 +5,7 @@ import json
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 import datetime
+import asyncio
 
 # Initialize FastAPI
 app = FastAPI()
@@ -81,6 +82,27 @@ async def chat_completion(request: Request):
             content={"error": str(e)}
         )
 
+# Synchronous function to generate response
+def generate_response(messages):
+    # Convert messages to model input format
+    prompt = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    
+    # Generate response
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=2048,
+        temperature=0.7,
+        top_p=0.95,
+        do_sample=True
+    )
+    
+    return tokenizer.decode(outputs[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
+
 # Gradio interface for testing
 def chat_interface(message, history):
     history = history or []
@@ -94,16 +116,12 @@ def chat_interface(message, history):
     # Add current message
     messages.append({"role": "user", "content": message})
     
-    # Create a mock request object with the messages
-    mock_request = Request(scope={"type": "http"}, receive=None)
-    mock_request.json = lambda: {"messages": messages}
-    
-    # Get response
-    response = await chat_completion(mock_request)
-    if isinstance(response, JSONResponse):
-        response_data = json.loads(response.body.decode())
-        return response_data["choices"][0]["message"]["content"]
-    return "Error generating response"
+    # Generate response synchronously
+    try:
+        response_text = generate_response(messages)
+        return response_text
+    except Exception as e:
+        return f"Error generating response: {str(e)}"
 
 interface = gr.ChatInterface(
     chat_interface,