Update app.py

app.py

@@ -103,6 +103,38 @@ def pdf_chat(pdf_file, text_query, temperature, top_p, max_output_tokens):
     except Exception as e:
         return f"Error processing the PDF: {str(e)}"
 
+# Function to process audio file and convert to text
+def process_audio(audio_file, query, temperature, top_p, max_output_tokens):
+    # Modified to handle numpy array or filepath depending on Audio component output
+    try:
+        if isinstance(audio_file, tuple):  # In case audio is returned as tuple (numpy array, sample rate)
+            # Convert numpy array to WAV bytes in memory
+            import numpy as np
+            import scipy.io.wavfile as wav
+            
+            audio_data, sample_rate = audio_file
+            buffer = io.BytesIO()
+            wav.write(buffer, sample_rate, audio_data)
+            buffer.seek(0)
+            audio_binary = buffer.read()
+        else:  # Filepath
+            with open(audio_file, "rb") as f:
+                audio_binary = f.read()
+                
+        # Transcribe the audio
+        transcription = transcribe_audio(audio_binary, api_key)
+        
+        # Use the transcription and query to get a response
+        messages = [
+            {"role": "user", "content": [
+                {"type": "text", "text": f"Transcription: {transcription}"},
+                {"type": "text", "text": f"Query: {query}"}
+            ]},
+        ]
+        return query_openai(messages, temperature, top_p, max_output_tokens)
+    except Exception as e:
+        return f"Error processing audio: {str(e)}"
+
 # Function to transcribe audio to text using OpenAI Whisper API
 def transcribe_audio(audio_binary, openai_api_key):
     if not openai_api_key:
@@ -121,9 +153,9 @@ def transcribe_audio(audio_binary, openai_api_key):
     except Exception as e:
         return f"Error transcribing audio: {str(e)}"
 
-# Function to clear the chat (Fix: Returns the correct number of outputs)
+# Function to clear the chat
 def clear_chat():
-    return "", "", "", "", "", "", "", None, "", None, "", 1.0, 1.0, 2048
+    return "", "", "", "", "", "", "", None, "", None, "", None, "", None, "", 1.0, 1.0, 2048
 
 # Gradio UI Layout
 with gr.Blocks() as demo:
@@ -176,14 +208,14 @@ with gr.Blocks() as demo:
             pdf_button = gr.Button("Ask", elem_id="ask_button")
         
         with gr.Tab("Voice Chat (Upload)"):
-            audio_upload = gr.File(label="Upload an Audio File", type="binary")
+            audio_upload = gr.File(label="Upload an Audio File")
             audio_query = gr.Textbox(label="Ask about the transcription")
             audio_output = gr.Textbox(label="Response", interactive=False)
             audio_button = gr.Button("Ask", elem_id="ask_button")
 
         with gr.Tab("Voice(Record) Chat"):
-            # Fix: Changed parameter name to match your Gradio version
-            audio_record = gr.Audio(label="Record your voice", type="binary")
+            # Fix: Changed type to "numpy" which is supported in your Gradio version
+            audio_record = gr.Audio(label="Record your voice", type="numpy")
             audio_record_query = gr.Textbox(label="Ask about the transcription")
             audio_record_output = gr.Textbox(label="Response", interactive=False)
             audio_record_button = gr.Button("Ask", elem_id="ask_button")
@@ -199,29 +231,25 @@ with gr.Blocks() as demo:
     pdf_button.click(pdf_chat, [pdf_upload, pdf_text_query, temperature, top_p, max_output_tokens], pdf_output)
     
     # For Voice Chat (Upload)
-    audio_button.click(
-        lambda audio_binary, query, temperature, top_p, max_output_tokens: query_openai(
-            [{"role": "user", "content": [{"type": "text", "text": transcribe_audio(audio_binary, api_key)}, {"type": "text", "text": query}]}],
-            temperature, top_p, max_output_tokens
-        ), [audio_upload, audio_query, temperature, top_p, max_output_tokens], audio_output
-    )
+    audio_button.click(process_audio, 
+                      [audio_upload, audio_query, temperature, top_p, max_output_tokens], 
+                      audio_output)
     
     # For Voice Chat (Record)
-    audio_record_button.click(
-        lambda audio_binary, query, temperature, top_p, max_output_tokens: query_openai(
-            [{"role": "user", "content": [{"type": "text", "text": transcribe_audio(audio_binary, api_key)}, {"type": "text", "text": query}]}],
-            temperature, top_p, max_output_tokens
-        ), [audio_record, audio_record_query, temperature, top_p, max_output_tokens], audio_record_output
-    )
+    audio_record_button.click(process_audio, 
+                             [audio_record, audio_record_query, temperature, top_p, max_output_tokens], 
+                             audio_record_output)
 
-    # Fix: Clear button resets all necessary fields correctly
+    # Clear button resets all necessary fields
     clear_button.click(
         clear_chat,
         outputs=[
             image_url, image_query, image_url_output, 
             text_query, text_output, 
             image_text_query, image_output, 
-            pdf_upload, pdf_text_query, pdf_output, 
+            pdf_upload, pdf_text_query, pdf_output,
+            audio_upload, audio_query, audio_output,
+            audio_record, audio_record_query, audio_record_output,
             temperature, top_p, max_output_tokens
         ]
     )