Create app.py

app.py

@@ -0,0 +1,83 @@
+import openai, gradio as gr, json, plotly.graph_objects as go
+from pathlib import Path
+
+SYSTEM_PROMPT = """You are a multimodal assistant. Return one of the following response types:
+- Plain text (just a natural reply)
+- JSON object: {"type":"image","prompt":"<dalle-3 prompt>"}
+- JSON object: {"type":"chart","title":"<title>","data":[{"x":[...],"y":[...],"label":"..."}]}
+- JSON object: {"type":"table","headers":["A","B"],"rows":[[1,2],[3,4]]}
+- JSON object: {"type":"audio","text":"Text to speak"}
+
+Respond in plain text unless image/chart/table/audio is clearly required.
+"""
+
+def build_messages(history, user_msg):
+    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+    for u, a in history:
+        messages.append({"role": "user", "content": u})
+        messages.append({"role": "assistant", "content": a})
+    messages.append({"role": "user", "content": user_msg})
+    return messages
+
+def multimodal_chat(api_key, user_msg, history):
+    if not api_key:
+        raise gr.Error("🔑 Please provide your OpenAI API key.")
+    openai.api_key = api_key
+    messages = build_messages(history, user_msg)
+    response = openai.chat.completions.create(model="gpt-4o", messages=messages)
+    content = response.choices[0].message.content.strip()
+
+    img_url, fig, table_html, audio_url = None, None, None, None
+    try:
+        parsed = json.loads(content)
+        t = parsed.get("type")
+        if t == "image":
+            img = openai.images.generate(model="dall-e-3", prompt=parsed["prompt"], size="1024x1024", n=1)
+            img_url = img.data[0].url
+            history.append([user_msg, f"![generated image]({img_url})"])
+        elif t == "chart":
+            fig = go.Figure()
+            for s in parsed["data"]:
+                fig.add_trace(go.Scatter(x=s["x"], y=s["y"], mode="lines+markers", name=s.get("label", "")))
+            fig.update_layout(title=parsed["title"])
+            history.append([user_msg, parsed["title"]])
+        elif t == "table":
+            headers = parsed["headers"]
+            rows = parsed["rows"]
+            table_html = f"<table><thead><tr>{''.join(f'<th>{h}</th>' for h in headers)}</tr></thead><tbody>"
+            table_html += ''.join(f"<tr>{''.join(f'<td>{c}</td>' for c in row)}</tr>" for row in rows)
+            table_html += "</tbody></table>"
+            history.append([user_msg, "Table generated below"])
+        elif t == "audio":
+            audio = openai.audio.speech.create(model="tts-1", voice="alloy", input=parsed["text"])
+            path = "/tmp/audio.mp3"
+            with open(path, "wb") as f: f.write(audio.read())
+            audio_url = path
+            history.append([user_msg, parsed["text"]])
+        else:
+            history.append([user_msg, content])
+    except Exception:
+        history.append([user_msg, content])
+    return history, img_url, fig, table_html, audio_url
+
+with gr.Blocks(css="style.css") as demo:
+    gr.Markdown("🤖 **Multimodal Assistant** – Text, Images, Charts, Tables, Audio", elem_id="zen-header")
+    api_key = gr.Textbox(label="OpenAI API Key", type="password", placeholder="sk-...")
+    chatbot = gr.Chatbot()
+    with gr.Row():
+        user_msg = gr.Textbox(label="Your message", scale=4)
+        send_btn = gr.Button("Send", variant="primary")
+    img_out = gr.Image()
+    chart_out = gr.Plot()
+    table_out = gr.HTML()
+    audio_out = gr.Audio(type="filepath")
+
+    def respond(api_key, user_msg, chat_history):
+        chat_history, img_url, fig, table, audio = multimodal_chat(api_key, user_msg, chat_history)
+        return chat_history, gr.update(value=img_url), gr.update(value=fig), gr.update(value=table), gr.update(value=audio)
+
+    send_btn.click(respond, [api_key, user_msg, chatbot], [chatbot, img_out, chart_out, table_out, audio_out])
+    user_msg.submit(respond, [api_key, user_msg, chatbot], [chatbot, img_out, chart_out, table_out, audio_out])
+
+if __name__ == "__main__":
+    demo.queue().launch()