Update app.py

app.py

@@ -11,6 +11,12 @@ import os
 from transformers import pipeline
 from transformers.pipelines.audio_utils import ffmpeg_read
 
+from langchain.prompts import PromptTemplate
+from langchain.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community import 
+from langchain_text_splitters import SentenceTransformersTokenTextSplitter
+
 from PIL import Image
 
 from transformers import LlavaNextProcessor, LlavaNextForConditionalGeneration, TextIteratorStreamer
@@ -18,6 +24,8 @@ processor = LlavaNextProcessor.from_pretrained("llava-hf/llava-v1.6-mistral-7b-h
 model = LlavaNextForConditionalGeneration.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf", torch_dtype=torch.float16, low_cpu_mem_usage=True)
 model.to("cuda:0")
 
+embeddings_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+
 ASR_MODEL_NAME = "openai/whisper-large-v3"
 ASR_BATCH_SIZE = 8
 ASR_CHUNK_LENGTH_S = 30
@@ -39,7 +47,7 @@ asr_pl = pipeline(
 )
 
 application_title = "Enlight Innovations Limited -- Demo"
-application_description = "This demo is desgined to illustrate our basic idea and feasibility in implementation."
+application_description = "This demo is desgined to illustrate our basic ideas and feasibility in implementation."
 
 @spaces.GPU
 def respond(
@@ -113,6 +121,7 @@ transcribe_interface = gr.Interface(
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 """
 chatbot_main = gr.Chatbot(label="Extraction Output")
+chatbot_main_input = gr.MultimodalTextbox({"text": "Choose the referred material(s) and ask your question.", "files":[]})
 chatbot_sys_output = gr.Textbox(value="You are a friendly Chatbot.", label="System Message")
 chatbot_max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max. New Tokens")
 chatbot_temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
@@ -126,9 +135,11 @@ chatbot_top_p = gr.Slider(
 
 chat_interface = gr.ChatInterface(
     respond,
+    multimodal=True,
     title=application_title,
     description=application_description,
     chatbot=chatbot_main,
+    textbox=chatbot_main_input,
     additional_inputs=[
         chatbot_sys_output,
         chatbot_max_tokens,
@@ -156,6 +167,10 @@ with gr.Blocks() as demo:
                              outputs=task_output
                             )
 
+    def on_selected_tab(selected_tab):
+        print(f"Selected tab: {selected_tab['value']}, Selected state: {selected_tab['selected']}")
+    demo.select(on_selected_tab)
+
 
 if __name__ == "__main__":
     demo.queue().launch() #demo.launch()