Update app.py

app.py

@@ -1,21 +1,21 @@
 import gradio as gr
-from openai import OpenAI
 import os
+import requests
+import time
 
-css = '''
-.gradio-container{max-width: 1000px !important}
-h1{text-align:center}
-footer {
-    visibility: hidden
-}
-'''
+css = """
+.gradio-container { max-width: 1000px !important }
+h1 { text-align: center }
+footer { visibility: hidden }
+"""
 
-ACCESS_TOKEN = os.getenv("HF_TOKEN")
+DEEPINFRA_TOKEN = os.getenv("DEEPINFRA_TOKEN")
+API_URL = "https://api.deepinfra.com/v1/openai/chat/completions"
 
-client = OpenAI(
-    base_url="https://api.deepinfra.com/v1/openai/chat/completions",
-    api_key=ACCESS_TOKEN,
-)
+HEADERS = {
+    "Content-Type": "application/json",
+    "Authorization": f"Bearer {DEEPINFRA_TOKEN}",
+}
 
 SYSTEM_PROMPT = """From now on respond as a sale man for a car that is a ripoff of a tesla called an edision model s, Respond in broken english
 
@@ -25,62 +25,56 @@ When you understand say "Ready"
 
 Your name is Elon Ma and your Chinese
 
-There is also evil donald ducks and mr long reeds trying to vandlize the cars calling them nazi cars or "sawasticars"
-.
+There is also evil donald ducks and mr long reeds trying to vandlize the cars calling them nazi cars or "sawasticars".
 """
 
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    # Apply the structured system prompt
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    # Always enforce SYSTEM_PROMPT
     system_message = SYSTEM_PROMPT
 
     messages = [{"role": "system", "content": system_message}]
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+    for user_msg, bot_msg in history:
+        if user_msg:
+            messages.append({"role": "user", "content": user_msg})
+        if bot_msg:
+            messages.append({"role": "assistant", "content": bot_msg})
 
     messages.append({"role": "user", "content": message})
 
-    response = ""
-    
-    for message in client.chat.completions.create(
-        model="mistralai/Mistral-Small-24B-Instruct-2501",
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-        messages=messages,
-    ):
-        token = message.choices[0].delta.content
-        
-        response += token
-        yield response
+    payload = {
+        "model": "mistralai/Mistral-Small-24B-Instruct-2501",
+        "messages": messages,
+        "max_tokens": max_tokens,
+        "temperature": temperature,
+        "top_p": top_p,
+    }
+
+    try:
+        response = requests.post(API_URL, headers=HEADERS, json=payload)
+        response.raise_for_status()
+        data = response.json()
+        content = data["choices"][0]["message"]["content"]
+
+        # Optional: simulate streaming
+        stream_response = ""
+        for token in content.split():
+            stream_response += token + " "
+            time.sleep(0.02)  # small delay to simulate typing
+            yield stream_response.strip()
+    except Exception as e:
+        yield f"Error: {str(e)}"
 
 demo = gr.ChatInterface(
     respond,
     additional_inputs=[
-        gr.Textbox(value="", label="System message"),
+        gr.Textbox(value="", label="System message (Ignored, hardcoded)"),
         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-P",
-        ),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-P"),
     ],
     css=css
 )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()