Update app.py

app.py

@@ -2,11 +2,10 @@ import gradio as gr
 import os
 import logging
 from langchain_core.prompts import ChatPromptTemplate
-from langchain.memory import ConversationBufferWindowMemory
 from langchain_core.output_parsers import StrOutputParser
+from langchain_openai import ChatOpenAI
 from langchain_community.graphs import Neo4jGraph
-from langchain_groq import ChatGroq
-from langchain.chains import GraphCypherQAChain
+from typing import List, Tuple
 from pydantic import BaseModel, Field
 from langchain_core.messages import AIMessage, HumanMessage
 from langchain_core.runnables import (
@@ -16,25 +15,25 @@ from langchain_core.runnables import (
     RunnableParallel,
 )
 from langchain_core.prompts.prompt import PromptTemplate
+import requests
 import tempfile
+from langchain.memory import ConversationBufferWindowMemory
 import time
-import threading
+import logging
+from langchain.chains import ConversationChain
 import torch
-import numpy as np
-import requests
+import torchaudio
 from transformers import pipeline, AutoModelForSpeechSeq2Seq, AutoProcessor
-from typing import List, Tuple
-
+import numpy as np
+import threading
 
-# Setup logging to a file to capture debug information
-logging.basicConfig(filename='neo4j_retrieval.log', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Setup for conversational memory
+#code for history
 conversational_memory = ConversationBufferWindowMemory(
-    memory_key='chat_history',
-    k=10,
-    return_messages=True
-)
+        memory_key='chat_history',
+        k=10,
+        return_messages=True
+    )
 
 # Setup Neo4j
 graph = Neo4jGraph(
@@ -43,10 +42,6 @@ graph = Neo4jGraph(
     password="Z10duoPkKCtENuOukw3eIlvl0xJWKtrVSr-_hGX1LQ4"
 )
 
-# Setup the Groq model
-groq_api_key = os.getenv('GROQ_API_KEY')
-llm = ChatGroq(groq_api_key=groq_api_key, model_name="Gemma2-9b-It")
-
 # Define entity extraction and retrieval functions
 class Entities(BaseModel):
     names: List[str] = Field(
@@ -58,7 +53,8 @@ entity_prompt = ChatPromptTemplate.from_messages([
     ("human", "Use the given format to extract information from the following input: {question}"),
 ])
 
-entity_chain = entity_prompt | llm.with_structured_output(Entities)
+chat_model = ChatOpenAI(temperature=0, model_name="gpt-4o", api_key=os.environ['OPENAI_API_KEY'])
+entity_chain = entity_prompt | chat_model.with_structured_output(Entities)
 
 def remove_lucene_chars(input: str) -> str:
     return input.translate(str.maketrans({
@@ -76,6 +72,9 @@ def generate_full_text_query(input: str) -> str:
     full_text_query += f" {words[-1]}~2"
     return full_text_query.strip()
 
+# Setup logging to a file to capture debug information
+logging.basicConfig(filename='neo4j_retrieval.log', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
+
 def structured_retriever(question: str) -> str:
     result = ""
     entities = entity_chain.invoke({"question": question})
@@ -104,7 +103,7 @@ def retriever_neo4j(question: str):
     logging.debug(f"Structured data: {structured_data}")
     return structured_data
 
-# Condense follow-up questions to standalone
+# Setup for condensing the follow-up questions
 _template = """Given the following conversation and a follow-up question, rephrase the follow-up question to be a standalone question,
 in its original language.
 Chat History:
@@ -130,19 +129,20 @@ _search_query = RunnableBranch(
             chat_history=lambda x: _format_chat_history(x["chat_history"])
         )
         | CONDENSE_QUESTION_PROMPT
-        | llm
+        | ChatOpenAI(temperature=0, api_key=os.environ['OPENAI_API_KEY'])
         | StrOutputParser(),
     ),
     RunnableLambda(lambda x: x["question"]),
 )
 
-# Define the prompt for response generation
+
 template = """I am a guide for Birmingham, Alabama. I can provide recommendations and insights about the city, including events and activities.
-Ask your question directly, and I'll provide a precise, short, and crisp response in a conversational way without any greeting.
+Ask your question directly, and I'll provide a precise and quick,short and crisp response in a conversational way without any Greet.
 {context}
 Question: {question}
 Answer:"""
 
+
 qa_prompt = ChatPromptTemplate.from_template(template)
 
 # Define the chain for Neo4j-based retrieval and response generation
@@ -154,7 +154,7 @@ chain_neo4j = (
         }
     )
     | qa_prompt
-    | llm
+    | chat_model
     | StrOutputParser()
 )
 
@@ -163,20 +163,31 @@ def get_response(question):
     try:
         return chain_neo4j.invoke({"question": question})
     except Exception as e:
-        logging.error(f"Error generating response: {str(e)}")
         return f"Error: {str(e)}"
 
 # Define the function to clear input and output
 def clear_fields():
-    return [], "", None
+    return [],"",None
 
 # Function to generate audio with Eleven Labs TTS
 def generate_audio_elevenlabs(text):
     XI_API_KEY = os.environ['ELEVENLABS_API']
     VOICE_ID = 'ehbJzYLQFpwbJmGkqbnW'
     tts_url = f"https://api.elevenlabs.io/v1/text-to-speech/{VOICE_ID}/stream"
-    headers = {"Accept": "application/json", "xi-api-key": XI_API_KEY}
-    data = {"text": str(text), "model_id": "eleven_multilingual_v2", "voice_settings": {"stability": 1.0}}
+    headers = {
+        "Accept": "application/json",
+        "xi-api-key": XI_API_KEY
+    }
+    data = {
+        "text": str(text),
+        "model_id": "eleven_multilingual_v2",
+        "voice_settings": {
+            "stability": 1.0,
+            "similarity_boost": 0.0,
+            "style": 0.60,
+            "use_speaker_boost": False
+        }
+    }
     response = requests.post(tts_url, headers=headers, json=data, stream=True)
     if response.ok:
         with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as f:
@@ -185,32 +196,175 @@ def generate_audio_elevenlabs(text):
                     f.write(chunk)
             audio_path = f.name
         logging.debug(f"Audio saved to {audio_path}")
-        return audio_path  # Return audio path for playback
+        return audio_path  # Return audio path for automatic playback
     else:
         logging.error(f"Error generating audio: {response.text}")
         return None
 
+# Function to add a user's message to the chat history and clear the input box
+def add_message(history, message):
+    if message.strip():
+        history.append((message, None))  # Add the user's message to the chat history only if it's not empty
+    return history, ""  # Clear the input box
+   
+# Define function to generate a streaming response
+def chat_with_bot(messages):
+    user_message = messages[-1][0]  # Get the last user message (input)
+    messages[-1] = (user_message, "")  # Prepare the placeholder for the bot's response
+    
+    response = get_response(user_message)
+    
+
+    
+    # Simulate streaming response by iterating over each character in the response
+    for character in response:
+        messages[-1] = (user_message, messages[-1][1] + character)
+        yield messages  # Stream each character
+        time.sleep(0.05)  # Adjust delay as needed for real-time effect
+
+    yield messages  # Final yield to ensure the full response is displayed
+
+
+# Function to generate audio with Eleven Labs TTS from the last bot response
+def generate_audio_from_last_response(history):
+    # Get the most recent bot response from the chat history
+    if history and len(history) > 0:
+        recent_response = history[-1][1]  # The second item in the tuple is the bot response text
+        if recent_response:
+            return generate_audio_elevenlabs(recent_response)
+    return None
+
+# Define example prompts
+examples = [
+    ["What are some popular events in Birmingham?"],
+    ["Who are the top players of the Crimson Tide?"],
+    ["Where can I find a hamburger?"],
+    ["What are some popular tourist attractions in Birmingham?"],
+    ["What are some good clubs in Birmingham?"],
+    ["Is there a farmer's market or craft fair in Birmingham, Alabama?"],
+    ["Are there any special holiday events or parades in Birmingham, Alabama, during December?"],
+    ["What are the best places to enjoy live music in Birmingham, Alabama?"]
+    
+]
+
+# Function to insert the prompt into the textbox when clicked
+def insert_prompt(current_text, prompt):
+    return prompt[0] if prompt else current_text
+
+
+# Define the ASR model with Whisper
+model_id = 'openai/whisper-large-v3'
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id, torch_dtype=torch_dtype).to(device)
+processor = AutoProcessor.from_pretrained(model_id)
+
+pipe_asr = pipeline(
+    "automatic-speech-recognition",
+    model=model,
+    tokenizer=processor.tokenizer,
+    feature_extractor=processor.feature_extractor,
+    max_new_tokens=128,
+    chunk_length_s=15,
+    batch_size=16,
+    torch_dtype=torch_dtype,
+    device=device,
+    return_timestamps=True
+)
+
+# Define the function to reset the state after 10 seconds
+def auto_reset_state():
+    time.sleep(5)
+    return None, ""  # Reset the state and clear input text
+
+
+def transcribe_function(stream, new_chunk):
+    try:
+        sr, y = new_chunk[0], new_chunk[1]
+    except TypeError:
+        print(f"Error chunk structure: {type(new_chunk)}, content: {new_chunk}")
+        return stream, "", None
+
+    # Ensure y is not empty and is at least 1-dimensional
+    if y is None or len(y) == 0:
+        return stream, "", None
+
+    y = y.astype(np.float32)
+    max_abs_y = np.max(np.abs(y))
+    if max_abs_y > 0:
+        y = y / max_abs_y
+
+    # Ensure stream is also at least 1-dimensional before concatenation
+    if stream is not None and len(stream) > 0:
+        stream = np.concatenate([stream, y])
+    else:
+        stream = y
+
+    # Process the audio data for transcription
+    result = pipe_asr({"array": stream, "sampling_rate": sr}, return_timestamps=False)
+    full_text = result.get("text", "")
+
+    # Start a thread to reset the state after 10 seconds
+    threading.Thread(target=auto_reset_state).start()
+
+    return stream, full_text, full_text
+
+
+
+# Define the function to clear the state and input text
+def clear_transcription_state():
+    return None, ""
+
+
+
 # Create the Gradio Blocks interface
 with gr.Blocks(theme="rawrsor1/Everforest") as demo:
     chatbot = gr.Chatbot([], elem_id="RADAR", bubble_full_width=False)
     with gr.Row():
         with gr.Column():
-            question_input = gr.Textbox(label="Ask a Question", placeholder="Type your question here...")
-        with gr.Column():
-            audio_output = gr.Audio(label="Audio", type="filepath", autoplay=True, interactive=False)
+           question_input = gr.Textbox(label="Ask a Question", placeholder="Type your question here...")
+           audio_input = gr.Audio(sources=["microphone"],streaming=True,type='numpy',every=0.1,label="Speak to Ask")
 
+            
+        
+        with gr.Column():
+            audio_output = gr.Audio(label="Audio", type="filepath",autoplay=True,interactive=False)
+    
     with gr.Row():
         with gr.Column():
             get_response_btn = gr.Button("Get Response")
+        with gr.Column():
+            clear_state_btn = gr.Button("Clear State")
         with gr.Column():
             generate_audio_btn = gr.Button("Generate Audio")
         with gr.Column():
-            clear_state_btn = gr.Button("Clear State")
+            clean_btn = gr.Button("Clean")
+
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("<h1 style='color: red;'>Example Prompts</h1>", elem_id="Example-Prompts")
+            gr.Examples(examples=examples, fn=insert_prompt, inputs=question_input, outputs=question_input,api_name="api_insert_example")
+
+    # Define interactions
+    # Define interactions for clicking the button
+    get_response_btn.click(fn=add_message, inputs=[chatbot, question_input], outputs=[chatbot, question_input],api_name="api_add_message_on_button_click")\
+                    .then(fn=chat_with_bot, inputs=[chatbot], outputs=chatbot,api_name="api_get response_on_button")
+    # Define interaction for hitting the Enter key
+    question_input.submit(fn=add_message, inputs=[chatbot, question_input], outputs=[chatbot, question_input],api_name="api_add_message_on _enter")\
+                  .then(fn=chat_with_bot, inputs=[chatbot], outputs=chatbot,api_name="api_get response_on_enter")
+
+    # Speech-to-Text functionality
+    state = gr.State()
+    audio_input.stream(transcribe_function, inputs=[state, audio_input], outputs=[state, question_input],api_name="api_voice_to_text")
+    
+    
+    generate_audio_btn.click(fn=generate_audio_from_last_response, inputs=chatbot, outputs=audio_output,api_name="api_generate_text_to_audio")
+    clean_btn.click(fn=clear_fields, inputs=[], outputs=[chatbot, question_input, audio_output],api_name="api_clear_textbox")
+
+
+     # Clear state interaction
+    clear_state_btn.click(fn=clear_transcription_state, outputs=[question_input, state],api_name="api_clean_state_transcription")
 
-    # Define interactions for buttons
-    get_response_btn.click(fn=get_response, inputs=question_input, outputs=chatbot)
-    generate_audio_btn.click(fn=generate_audio_elevenlabs, inputs=chatbot, outputs=audio_output)
-    clear_state_btn.click(fn=clear_fields, outputs=[chatbot, question_input, audio_output])
 
 # Launch the Gradio interface
-demo.launch(show_error=True)
+demo.launch(show_error=True)