audio input is working "seamlessly"

query.py

@@ -8,7 +8,10 @@ from langchain_core.runnables import RunnablePassthrough, RunnableLambda
 from langchain_core.messages import SystemMessage, AIMessage, HumanMessage
 from langchain_openai import OpenAIEmbeddings, ChatOpenAI
 
+from openai import OpenAI
+
 from json import loads as json_loads
+import time
 import os
 
 prompt_template = os.environ.get("PROMPT_TEMPLATE")
@@ -18,10 +21,11 @@ prompt = ChatPromptTemplate.from_messages([('system', prompt_template)])
 AI = True
 
 def ai_setup():
-    global llm, prompt_chain
-    llm = ChatOpenAI(model = "gpt-4o", temperature=0.8)
+    global llm, prompt_chain, oai_client
     
     if AI:
+        oai_client = OpenAI()
+        llm = ChatOpenAI(model = "gpt-4o", temperature=0.8)
         embedding = OpenAIEmbeddings()
         vstore = AstraDBVectorStore(
             embedding=embedding,
@@ -97,8 +101,8 @@ def new_state():
     })
 
 def auth(token, state):
-    tokens=os.environ.get("APP_TOKENS", None)
-    if tokens is None:
+    tokens=os.environ.get("APP_TOKENS")
+    if not tokens:
         state["user"] = "anonymous"
     else:
         tokens=json_loads(tokens)
@@ -118,7 +122,7 @@ def chat(message, history, state):
     if (state is None) or (not state['user']):
         gr.Warning("You need to authenticate first")
         yield "You need to authenticate first"
-    else:
+    elif AI:
         if not history:
             system_prompt = prompt_chain.invoke(message)
             system_prompt = system_prompt.messages[0]
@@ -136,6 +140,22 @@ def chat(message, history, state):
         for response in llm.stream(messages):
             all += response.content
             yield all
+    else:
+        yield f"{time.ctime()}: You said: {message}"
+
+def on_audio(path):
+    if not path:
+        return [gr.update(), None]
+    if AI:
+        text = oai_client.audio.transcriptions.create(
+                model="whisper-1", 
+                file=open(path, "rb"),
+                response_format="text"
+            )
+    else:
+        text = f"{time.ctime()}: You said something"
+
+    return (text, None)
 
 def gr_main():
     theme = gr.Theme.from_hub("freddyaboulton/[email protected]")
@@ -151,10 +171,19 @@ def gr_main():
         theme=theme
         ) as app:
             state = new_state()
-            gr.ChatInterface(
+            chatbot = gr.Chatbot(show_label=False, render=False, scale=1)
+            iface = gr.ChatInterface(
                 chat,
-                chatbot=gr.Chatbot(show_label=False, render=False, scale=1),
+                chatbot=chatbot,
                 title="Sherlock Holmes stories",
+                submit_btn=gr.Button(
+                                "Submit",
+                                variant="primary",
+                                scale=1,
+                                min_width=150,
+                                elem_id="submit_btn",
+                                render=False
+                            ),
                 examples=[
                     ["I arrived late last night and found a dead goose in my bed"],
                     ["Help please sir. I'm about to get married, to the most lovely lady,"
@@ -162,6 +191,21 @@ def gr_main():
                     "of my past I'd rather keep quiet, unless I don't marry"],
                 ],
                 additional_inputs=[state])
+            
+            mic = gr.Audio(
+                sources=["microphone"],
+                type="filepath",
+                show_label=False,
+                format="mp3",
+                
+                waveform_options=gr.WaveformOptions(sample_rate=16000))
+            mic.change(
+                    on_audio, [mic], [iface.textbox, mic]
+                ).then(
+                    lambda x:None,
+                    js='function (text){debugger; if (text) document.getElementById("submit_btn").click(); return [text]}',
+                    inputs=iface.textbox
+                )
             token = gr.Textbox(visible=False)
             app.load(auth,
                 [token,state],
@@ -172,4 +216,7 @@ def gr_main():
     
 if __name__ == "__main__":
     ai_setup()
-    gr_main()
+    gr_main()
+    
+# Autoplay audio
+# https://github.com/gradio-app/gradio/issues/1349