Update app.py

app.py

@@ -12,11 +12,14 @@ app = FastAPI()
 # Load model and tokenizer
 model_name = "Qwen/Qwen2.5-Coder-32B"
 tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+# Configure model loading with specific parameters
 model = AutoModelForCausalLM.from_pretrained(
     model_name,
     device_map="auto",
     trust_remote_code=True,
-    torch_dtype=torch.float16
+    torch_dtype=torch.float16,
+    low_cpu_mem_usage=True
 )
 
 def format_chat_response(response_text, prompt_tokens, completion_tokens):
@@ -91,12 +94,15 @@ def chat_interface(message, history):
     # Add current message
     messages.append({"role": "user", "content": message})
     
+    # Create a mock request object with the messages
+    mock_request = Request(scope={"type": "http"}, receive=None)
+    mock_request.json = lambda: {"messages": messages}
+    
     # Get response
-    response = chat_completion(Request(scope={"type": "http"}, receive=None))
+    response = await chat_completion(mock_request)
     if isinstance(response, JSONResponse):
-        response_data = response.body.decode()
-        response_json = json.loads(response_data)
-        return response_json["choices"][0]["message"]["content"]
+        response_data = json.loads(response.body.decode())
+        return response_data["choices"][0]["message"]["content"]
     return "Error generating response"
 
 interface = gr.ChatInterface(