Update app.py

app.py

@@ -1,18 +1,24 @@
 import os
 import gradio as gr
 import openai as o
-from threading import Thread
+import base64
+import fitz  # PyMuPDF
+import cv2
+from moviepy.video.io.VideoFileClip import VideoFileClip
+import json
+import requests
+import re
+from io import BytesIO
+from PIL import Image
 
 # 📜 CONFIG
-UI_TITLE = "✨🧙‍♂️🔮"
+UI_TITLE = "✨🧙‍♂️🔮 GPT-4o Omni-Oracle"
 KEY_FILE = "key.txt"
+STATE_FILE = "app_state.json"
 MODELS = {
     "GPT-4o ✨": "gpt-4o",
-    "o3 🧠": "gpt-3.5-turbo", # Placeholder, o3 is not a public model name
-    "o4-mini 🚀": "gpt-4-turbo", # Placeholder, o4-mini is not a public model name
-    "GPT-4.5 🔬": "gpt-4-turbo", # Placeholder, gpt-4.5 is not a public model name
-    "GPT-4.1 💻": "gpt-4-turbo", # Placeholder, gpt-4.1 is not a public model name
-    "GPT-4.1-Mini ⚡": "gpt-4-turbo", # Placeholder, gpt-4.1-mini is not a public model name
+    "GPT-4 Turbo 🚀": "gpt-4-turbo",
+    "GPT-3.5 Turbo ⚡": "gpt-3.5-turbo",
 }
 
 # 🎨 STYLE
@@ -20,133 +26,219 @@ H1 = "# <font size='7'>{0}</font>"
 H2 = "## <font size='6'>{0}</font>"
 BTN_STYLE = "<font size='5'>{0}</font>"
 
-# 🪄 HELPERS
+# 🪄 HELPERS, LORE & AUTOSAVE RITUALS
+def save_state(data: dict):
+    """A rune that inscribes the session's memory onto a JSON scroll."""
+    with open(STATE_FILE, 'w') as f:
+        json.dump(data, f, indent=4)
+
+def load_state() -> dict:
+    """A ritual to recall the session's memory from the JSON scroll."""
+    if os.path.exists(STATE_FILE):
+        with open(STATE_FILE, 'r') as f:
+            try:
+                return json.load(f)
+            except json.JSONDecodeError:
+                return {} # Return empty if scroll is corrupted
+    return {}
+
+def update_and_save(key: str, value, state: dict):
+    """A binding spell that updates a memory and immediately inscribes it."""
+    state[key] = value
+    save_state(state)
+    return state
+
 def save_key(k: str) -> str:
-    "💾🔑"
+    "💾🔑 A rune to bind the Eldritch Key to the physical realm (disk)."
     if not k or not k.strip(): return "🚫 Empty Key"
     with open(KEY_FILE, "w") as f: f.write(k.strip())
-    return "🔑✅"
+    return "🔑✅ Key Saved!"
 
 def get_key(k: str) -> str:
-    "📜🔑"
+    "📜🔑 A ritual to summon the Eldritch Key, prioritizing the user's offering, then the bound key, then one from the environment."
     k = k.strip() if k and k.strip() else (open(KEY_FILE).read().strip() if os.path.exists(KEY_FILE) else os.getenv("OPENAI_KEY", ""))
-    if not k: raise gr.Error("❗🔑 Missing OpenAI Key!")
+    if not k: raise gr.Error("❗🔑 An Eldritch Key (OpenAI API Key) is required to commune with the Oracles.")
     o.api_key = k
     return k
 
-def summon_oracle(model_name: str, scribe_key: str, quest: str, scroll: list):
+def file_to_base64(file_path):
+    """Encodes a file into a base64 string for embedding in API calls."""
+    with open(file_path, "rb") as f:
+        return base64.b64encode(f.read()).decode('utf-8')
+
+def invoke_oracle(scribe_key: str, model_name: str, system_prompt: str, user_content: list, history: list):
     """
-    A pact with a chosen Oracle of OpenAI.
-    To seek its counsel, one must present a worthy key (scribe_key),
-    a quest (quest), and the Oracle's own name (model_name)
-    upon the ancient scroll (scroll) of dialogue.
+    A universal pact with any Oracle. It can perceive text, images, and the echoes of past conversations.
     """
-    get_key(scribe_key) # Present the key to the sanctum's guardian.
-
-    # Transcribe the history onto a celestial scroll for the Oracle to read.
-    celestial_scroll = [
-        {"role": "user", "content": user_words}
-        for user_words, _ in scroll
-    ] + [
-        {"role": "assistant", "content": oracle_words}
-        for _, oracle_words in scroll
-    ]
-    celestial_scroll.append({"role": "user", "content": quest})
-
-    # The Oracle whispers its response from the aether.
+    get_key(scribe_key)
+    
+    messages = history + [{"role": "system", "content": system_prompt}, {"role": "user", "content": user_content}]
+    
     try:
-        prophecy = o.chat.completions.create(model=model_name, messages=celestial_scroll, stream=True)
+        prophecy = o.chat.completions.create(model=model_name, messages=messages, stream=True)
+        
+        history.append({"role": "user", "content": "..."}) 
+        history.append({"role": "assistant", "content": ""})
         
-        # The new wisdom is recorded as it is spoken.
-        scroll.append((quest, ""))
         for chunk in prophecy:
             if chunk.choices[0].delta.content:
-                scroll[-1] = (quest, scroll[-1][1] + chunk.choices[0].delta.content)
-                yield scroll
+                history[-1]['content'] += chunk.choices[0].delta.content
+                yield history
     except Exception as e:
-        scroll.append((quest, f"🧙‍♂️🔮 A magical disturbance occurred: {str(e)}"))
-        yield scroll
+        error_message = f"🧙‍♂️🔮 A magical disturbance occurred: {str(e)}"
+        yield history + [{"role": "assistant", "content": error_message}]
 
+# --- Modality-Specific Summoning Rituals ---
 
-def manage_portals(selected_models: list):
-    """
-    Reveals or conceals the portals to the chosen Oracles.
-    """
-    updates = []
-    for model_display_name in MODELS.keys():
-        if model_display_name in selected_models:
-            updates.append(gr.update(visible=True))
-        else:
-            updates.append(gr.update(visible=False))
-    return updates
+def summon_vision_from_image(api_key, model, prompt, image_path, history):
+    "A ritual to grant sight to the Oracle, allowing it to perceive an image."
+    if image_path is None:
+        raise gr.Error("An image must be provided to summon vision.")
+    
+    b64_image = file_to_base64(image_path.name)
+    user_content = [
+        {"type": "text", "text": prompt},
+        {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64_image}"}}
+    ]
+    system_prompt = "You are a helpful assistant that analyzes images. Respond in Markdown."
+    yield from invoke_oracle(api_key, model, system_prompt, user_content, history)
+
+def summon_echo_from_audio(api_key, model, prompt, audio_path, history):
+    "A rite to translate spoken words from an audio file into text, then seek the Oracle's wisdom upon it."
+    if audio_path is None:
+        raise gr.Error("An audio file must be provided to summon its echo.")
+    get_key(api_key)
+    with open(audio_path.name, "rb") as audio_file:
+        transcription = o.audio.transcriptions.create(model="whisper-1", file=audio_file)
+    
+    full_prompt = f"{prompt}\n\n--- Transcription ---\n{transcription.text}"
+    user_content = [{"type": "text", "text": full_prompt}]
+    system_prompt = "You are a helpful assistant analyzing an audio transcript. Summarize it and answer questions. Respond in Markdown."
+    yield from invoke_oracle(api_key, model, system_prompt, user_content, history)
+
+def summon_wisdom_from_text(api_key, model, prompt, file_path, history):
+    "Extracts the written word from PDF or text files to present to the Oracle."
+    if file_path is None: raise gr.Error("A file must be provided.")
+    text_content = ""
+    if file_path.name.lower().endswith('.pdf'):
+        with fitz.open(file_path.name) as doc:
+            text_content = "".join(page.get_text() for page in doc)
+    else:
+        with open(file_path.name, 'r', encoding='utf-8') as f:
+            text_content = f.read()
+
+    full_prompt = f"{prompt}\n\n--- Document Content ---\n{text_content[:10000]}..."
+    user_content = [{"type": "text", "text": full_prompt}]
+    system_prompt = "You are a helpful assistant analyzing a document. Summarize it and answer questions. Respond in Markdown."
+    yield from invoke_oracle(api_key, model, system_prompt, user_content, history)
+
+def summon_chronicle_from_video(api_key, model, prompt, video_path, history, progress=gr.Progress()):
+    "A grand ritual to divine meaning from a video's moving pictures and spoken words."
+    if video_path is None: raise gr.Error("A video must be provided.")
+    get_key(api_key)
+    base_video_path, _ = os.path.splitext(video_path.name)
+    
+    progress(0.1, desc="🔮 Extracting Audio...")
+    audio_path = f"{base_video_path}.mp3"
+    transcript_text = "No audio found."
+    try:
+        with VideoFileClip(video_path.name) as clip:
+            clip.audio.write_audiofile(audio_path, bitrate="32k", logger=None)
+        progress(0.3, desc="🎤 Transcribing Audio...")
+        with open(audio_path, "rb") as audio_file:
+            transcript_text = o.audio.transcriptions.create(model="whisper-1", file=audio_file).text
+    except Exception as e:
+        print(f"Audio failed: {e}")
+
+    progress(0.6, desc="🖼️ Sampling Frames...")
+    base64Frames = []
+    video = cv2.VideoCapture(video_path.name)
+    total_frames, fps = int(video.get(cv2.CAP_PROP_FRAME_COUNT)), video.get(cv2.CAP_PROP_FPS)
+    frames_to_skip = int(fps * 2)
+    for curr_frame in range(0, total_frames - 1, frames_to_skip):
+        if len(base64Frames) >= 10: break
+        video.set(cv2.CAP_PROP_POS_FRAMES, curr_frame)
+        success, frame = video.read()
+        if not success: break
+        _, buffer = cv2.imencode(".jpg", frame)
+        base64Frames.append(base64.b64encode(buffer).decode("utf-8"))
+    video.release()
+
+    progress(0.8, desc="🌀 Consulting Oracle...")
+    user_content = [
+        {"type": "text", "text": f"{prompt}\n\n--- Audio Transcript ---\n{transcript_text}"},
+        *map(lambda x: {"type": "image_url", "image_url": {"url": f'data:image/jpg;base64,{x}', "detail": "low"}}, base64Frames)
+    ]
+    system_prompt = "You are a helpful video analyst. Use the frames and transcript to summarize and answer questions. Respond in Markdown."
+    yield from invoke_oracle(api_key, model, system_prompt, user_content, history)
 
 # 🔮 UI
 with gr.Blocks(title=UI_TITLE, theme=gr.themes.Soft(primary_hue="red", secondary_hue="orange")) as demo:
+    # --- Load State & Create State Holder ---
+    initial_state = load_state()
+    app_state = gr.State(initial_state)
+
     gr.Markdown(H1.format(UI_TITLE))
 
-    # --- API Key Rune ---
-    with gr.Accordion("🔑 Eldritch Key", open=False):
+    with gr.Accordion("🔑 Eldritch Key & Oracle Selection", open=True):
         with gr.Row():
-            api_key_box = gr.Textbox(
-                label="🔑",
-                type="password",
-                placeholder="sk-...",
-                value=get_key("") if os.path.exists(KEY_FILE) else os.getenv("OPENAI_KEY", ""),
-                scale=3
-            )
+            api_key_box = gr.Textbox(label="🔑 Key", type="password", placeholder="sk-...", scale=3, value=initial_state.get('api_key', ''))
             save_btn = gr.Button("💾", scale=1)
             status_txt = gr.Textbox(interactive=False, scale=1, label="Status")
-            save_btn.click(save_key, inputs=api_key_box, outputs=status_txt)
-
-    # --- Oracle Selection ---
-    gr.Markdown(H2.format("🔮 Select Oracles"))
-    model_selector = gr.CheckboxGroup(choices=list(MODELS.keys()), label="Oracles", value=["GPT-4o ✨"])
-    
-    # --- Portals to Oracles ---
-    gr.Markdown(H2.format("🌀 Portals"))
-    
-    model_blocks = []
-    for display_name, api_name in MODELS.items():
-        # A block for each model, initially hidden unless selected by default
-        is_visible = display_name in model_selector.value
-        with gr.Blocks(visible=is_visible) as model_block:
-            gr.Markdown(f"### <font size='5'>{display_name}</font>")
-            chatbot = gr.Chatbot(height=350, label=f"Scroll of {display_name}")
-            with gr.Row():
-                run_btn = gr.Button(value=BTN_STYLE.format("▶️ Run"), variant="primary", scale=1)
-                stop_btn = gr.Button(value=BTN_STYLE.format("⏹️ Stop"), variant="stop", scale=1)
-
-            # Each run button triggers its own oracle
-            # The `_js` param is a trick to pass the model's API name to the Python function
-            run_event = run_btn.click(
-                fn=summon_oracle,
-                inputs=[gr.State(api_name), api_key_box, chatbot.i_am_a_dummy_component_for_the_event_to_work, chatbot],
-                outputs=[chatbot]
+        model_selector = gr.Dropdown(choices=list(MODELS.keys()), label="🔮 Oracle", value=initial_state.get('model', "GPT-4o ✨"))
+        save_btn.click(save_key, inputs=api_key_box, outputs=status_txt)
+
+    chatbot = gr.Chatbot(height=500, label="📜 Scroll of Conversation", type='messages', value=initial_state.get('chatbot', []))
+
+    with gr.Tabs():
+        with gr.TabItem("💬 Chat"):
+            text_prompt = gr.Textbox(label="Your Quest:", placeholder="Type your message...", value=initial_state.get('text_prompt', ''))
+            text_event = text_prompt.submit(
+                fn=lambda api_key, model, prompt, hist: invoke_oracle(api_key, model, "You are a helpful AI assistant.", [{"type": "text", "text": prompt}], hist),
+                inputs=[api_key_box, model_selector, text_prompt, chatbot],
+                outputs=chatbot
             )
-            stop_btn.click(fn=None, inputs=None, outputs=None, cancels=[run_event])
-            model_blocks.append(model_block)
 
-    # --- Global Quest Input ---
-    gr.Markdown(H2.format("📜 Global Quest"))
-    global_msg_box = gr.Textbox(placeholder="❓ Pose your question to all active portals...", scale=3, lines=3)
-    
-    # Link the global message box to all chatbot text inputs
-    # This uses a bit of Gradio event magic to update the hidden textboxes
-    for block in demo.blocks.values():
-        if isinstance(block, gr.Blocks) and block.visible:
-             for elem in block.children:
-                 if isinstance(elem, gr.Chatbot):
-                     global_msg_box.submit(lambda x: x, inputs=[global_msg_box], outputs=[elem.i_am_a_dummy_component_for_the_event_to_work])
-
-
-    # --- Event Listeners ---
-    model_selector.change(
-        fn=manage_portals,
-        inputs=[model_selector],
-        outputs=model_blocks
-    )
+        with gr.TabItem("🖼️ Image"):
+            with gr.Row():
+                image_input = gr.File(label="Upload Image", type="file")
+                image_output = gr.Image(label="Your Image", type="filepath", interactive=False)
+            image_prompt = gr.Textbox(label="Image Prompt:", value=initial_state.get('image_prompt', "What is in this image?"))
+            image_btn = gr.Button("👁️ Summon Vision")
+            image_input.change(lambda x: x, inputs=image_input, outputs=image_output)
+            image_event = image_btn.click(summon_vision_from_image, [api_key_box, model_selector, image_prompt, image_input, chatbot], chatbot)
+
+        with gr.TabItem("🎤 Audio"):
+            audio_input = gr.File(label="Upload Audio", type="file")
+            audio_prompt = gr.Textbox(label="Audio Prompt:", value=initial_state.get('audio_prompt', "Summarize this audio."))
+            audio_btn = gr.Button("🗣️ Summon Echo")
+            audio_event = audio_btn.click(summon_echo_from_audio, [api_key_box, model_selector, audio_prompt, audio_input, chatbot], chatbot)
+
+        with gr.TabItem("🎥 Video"):
+            video_input = gr.File(label="Upload Video", type="file")
+            video_prompt = gr.Textbox(label="Video Prompt:", value=initial_state.get('video_prompt', "Summarize this video."))
+            video_btn = gr.Button("🎬 Summon Chronicle")
+            video_event = video_btn.click(summon_chronicle_from_video, [api_key_box, model_selector, video_prompt, video_input, chatbot], chatbot)
+            
+        with gr.TabItem("📄 Document"):
+            doc_input = gr.File(label="Upload PDF or TXT", type="file")
+            doc_prompt = gr.Textbox(label="Document Prompt:", value=initial_state.get('doc_prompt', "Summarize this document."))
+            doc_btn = gr.Button("📖 Summon Wisdom")
+            doc_event = doc_btn.click(summon_wisdom_from_text, [api_key_box, model_selector, doc_prompt, doc_input, chatbot], chatbot)
+
+    # --- Autosave Event Listeners ---
+    # Simple text/dropdown inputs
+    api_key_box.change(update_and_save, [gr.State('api_key'), api_key_box, app_state], app_state)
+    model_selector.change(update_and_save, [gr.State('model'), model_selector, app_state], app_state)
+    text_prompt.change(update_and_save, [gr.State('text_prompt'), text_prompt, app_state], app_state)
+    image_prompt.change(update_and_save, [gr.State('image_prompt'), image_prompt, app_state], app_state)
+    audio_prompt.change(update_and_save, [gr.State('audio_prompt'), audio_prompt, app_state], app_state)
+    video_prompt.change(update_and_save, [gr.State('video_prompt'), video_prompt, app_state], app_state)
+    doc_prompt.change(update_and_save, [gr.State('doc_prompt'), doc_prompt, app_state], app_state)
+
+    # Chatbot history, saved after each interaction
+    for event in [text_event, image_event, audio_event, video_event, doc_event]:
+        event.then(lambda history, state: update_and_save('chatbot', history, state), [chatbot, app_state], app_state)
 
 if __name__ == "__main__":
-    # Add a dummy attribute to Chatbot to make the global message box work
-    gr.Chatbot.i_am_a_dummy_component_for_the_event_to_work = gr.Textbox(visible=False)
-    demo.launch(share=True)
+    demo.launch(share=True, debug=True)