Update app.py

app.py

@@ -1,105 +1,100 @@
-import torch
-import json
-from torch import cuda, bfloat16
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, BitsAndBytesConfig, StoppingCriteria, StoppingCriteriaList
-from langchain.llms import HuggingFacePipeline
+import os
+from langchain.document_loaders import TextLoader, DirectoryLoader
 from langchain.vectorstores import FAISS
-from langchain.chains import ConversationalRetrievalChain
+from sentence_transformers import SentenceTransformer
+import faiss
+import torch
+import numpy as np
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
+from datetime import datetime
 import gradio as gr
-from langchain.embeddings import HuggingFaceEmbeddings
-import os
 
-class Chatbot:
-    def __init__(self):
-        self.HF_TOKEN = os.environ.get("HF_TOKEN", None)
-        self.model_id = "mistralai/Mistral-7B-Instruct-v0.2"
-        self.device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'
-        self.bnb_config = BitsAndBytesConfig(
+class DocumentRetrievalAndGeneration:
+    def __init__(self, embedding_model_name, lm_model_id, data_folder, faiss_index_path):
+        self.documents = self.load_documents(data_folder)
+        self.embeddings = SentenceTransformer(embedding_model_name)
+        self.gpu_index = self.load_faiss_index(faiss_index_path)
+        self.llm = self.initialize_llm(lm_model_id)
+
+    def load_documents(self, folder_path):
+        loader = DirectoryLoader(folder_path, loader_cls=TextLoader)
+        documents = loader.load()
+        print('Length of documents:', len(documents))
+        return documents
+
+    def load_faiss_index(self, faiss_index_path):
+        cpu_index = faiss.read_index(faiss_index_path)
+        gpu_resource = faiss.StandardGpuResources()
+        gpu_index = faiss.index_cpu_to_gpu(gpu_resource, 0, cpu_index)
+        return gpu_index
+
+    def initialize_llm(self, model_id):
+        bnb_config = BitsAndBytesConfig(
             load_in_4bit=True,
-            bnb_4bit_quant_type='nf4',
             bnb_4bit_use_double_quant=True,
-            bnb_4bit_compute_dtype=bfloat16
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16
         )
-        self.tokenizer = AutoTokenizer.from_pretrained(self.model_id, token=self.HF_TOKEN)
-        self.model = AutoModelForCausalLM.from_pretrained(self.model_id, device_map="auto", token=self.HF_TOKEN, quantization_config=self.bnb_config)
-        self.stop_list = ['\nHuman:', '\n```\n']
-        self.stop_token_ids = [self.tokenizer(x)['input_ids'] for x in self.stop_list]
-        self.stop_token_ids = [torch.LongTensor(x).to(self.device) for x in self.stop_token_ids]
-        self.stopping_criteria = StoppingCriteriaList([self.StopOnTokens()])
-
-        self.generate_text = pipeline(
-            model=self.model,
-            tokenizer=self.tokenizer,
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config)
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        generate_text = pipeline(
+            model=model,
+            tokenizer=tokenizer,
             return_full_text=True,
             task='text-generation',
-            temperature=0.1,
+            temperature=0.6,
             max_new_tokens=2048,
         )
-        self.llm = HuggingFacePipeline(pipeline=self.generate_text)
-
-        try:
-            self.vectorstore = FAISS.load_local('faiss_index', HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={"device": "cuda"}))
-            # self.vectorstore = FAISS.load_local('faiss_index', HuggingFaceEmbeddings(model_name="flax-sentence-embeddings/all_datasets_v3_MiniLM-L12", model_kwargs={"device": "cuda"}))
-            print("Loaded embedding successfully")
-        except ImportError as e:
-            print("FAISS could not be imported. Make sure FAISS is installed correctly.")
-            raise e
-
-        self.chain = ConversationalRetrievalChain.from_llm(self.llm, self.vectorstore.as_retriever(), return_source_documents=True)
-        self.chat_history = []
-
-    class StopOnTokens(StoppingCriteria):
-        def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
-            for stop_ids in self.stop_token_ids:
-                if torch.eq(input_ids[0][-len(stop_ids):], stop_ids).all():
-                    return True
-            return False
-
-    def format_prompt(self, query):
-        prompt=f"""
-        You are a knowledgeable assistant with access to a comprehensive database. 
-        I need you to answer my question and provide related information in a specific format.
-        I have provided four relatable json files , choose the most suitable chunks for answering the query
-        Here's what I need:
-        Include a final answer without additional comments, sign-offs, or extra phrases. Be direct and to the point.
-        
-        Here's my question:
-        Query:{query}
-        Solution==>
-        Example1
-        Query: "How to use IPU1_0 instead of A15_0 to process NDK in TDA2x-EVM",
-        Solution: "To use IPU1_0 instead of A15_0 to process NDK in TDA2x-EVM, you need to modify the configuration file of the NDK application. Specifically, change the processor reference from 'A15_0' to 'IPU1_0'.",
-        
-        Example2
-        Query: "Can BQ25896 support I2C interface?",
-        Solution: "Yes, the BQ25896 charger supports the I2C interface for communication.",
-        """
-        return prompt
+        return generate_text
+
+    def query_and_generate_response(self, query):
+        query_embedding = self.embeddings.encode(query, convert_to_tensor=True).cpu().numpy()
+        distances, indices = self.gpu_index.search(np.array([query_embedding]), k=5)
 
-    def qa_infer(self, query):
         content = ""
-        formatted_prompt = self.format_prompt(query)
-        result = self.chain({"question": formatted_prompt, "chat_history": self.chat_history})
-        for doc in result['source_documents']:
+        for idx in indices[0]:
             content += "-" * 50 + "\n"
-            content += doc.page_content + "\n"
-        print(content)
-        print("#" * 100)
-        print(result['answer'])
-
-        output_file = "output.txt"
-        with open(output_file, "w") as f:
-            f.write("Query:\n")
-            f.write(query + "\n\n")
-            f.write("Answer:\n")
-            f.write(result['answer'] + "\n\n")
-            f.write("Source Documents:\n")
-            f.write(content + "\n")
-
-        download_link = f'<a href="file/{output_file}" download>Download Output File</a>'
-        return result['answer'], content, download_link
-
-    def launch_interface(self):
+            content += self.documents[idx].page_content + "\n"
+            print(self.documents[idx].page_content)
+            print("############################")
+        
+        prompt = f"Query: {query}\nSolution: {content}\n"
+
+        # Encode and prepare inputs
+        messages = [{"role": "user", "content": prompt}]
+        encodeds = self.llm.tokenizer.apply_chat_template(messages, return_tensors="pt")
+        model_inputs = encodeds.to(self.llm.device)
+
+        # Perform inference and measure time
+        start_time = datetime.now()
+        generated_ids = self.llm.model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
+        elapsed_time = datetime.now() - start_time
+
+        # Decode and return output
+        decoded = self.llm.tokenizer.batch_decode(generated_ids)
+        generated_response = decoded[0]
+        print("Generated response:", generated_response)
+        print("Time elapsed:", elapsed_time)
+        print("Device in use:", self.llm.device)
+
+        return generated_response
+
+    def qa_infer_gradio(self, query):
+        response = self.query_and_generate_response(query)
+        return response
+
+if __name__ == "__main__":
+    # Example usage
+    embedding_model_name = 'flax-sentence-embeddings/all_datasets_v3_MiniLM-L12'
+    lm_model_id = "mistralai/Mistral-7B-Instruct-v0.2"
+    data_folder = 'sample_embedding_folder'
+    faiss_index_path = 'faiss_index_new_model3.index'
+
+    doc_retrieval_gen = DocumentRetrievalAndGeneration(embedding_model_name, lm_model_id, data_folder, faiss_index_path)
+
+    # Define Gradio interface function
+    def launch_interface():
         css_code = """
             .gradio-container {
                 background-color: #daccdb;
@@ -119,8 +114,6 @@ class Chatbot:
                     "I'm using Code Composer Studio 5.4.0.00091 and enabled FPv4SPD16 floating point support for CortexM4 in TDA2. However, after building the project, the .asm file shows --float_support=vfplib instead of FPv4SPD16. Why is this happening?", 
                     "Master core in TDA2XX is a15 and in TDA3XX it is m4,so we have to shift all modules that are being used by a15 in TDA2XX to m4 in TDA3xx."]
         
-        
-
         file_path = "ticketNames.txt"
 
         # Read the file content
@@ -129,43 +122,19 @@ class Chatbot:
         ticket_names = json.loads(content)
         dropdown = gr.Dropdown(label="Sample queries", choices=ticket_names)
         
-        tab1 = gr.Interface(fn=self.qa_infer, inputs=[gr.Textbox(label="QUERY", placeholder ="Enter your query here")], allow_flagging='never', examples=EXAMPLES, cache_examples=False, outputs=[gr.Textbox(label="SOLUTION"), gr.Textbox(label="RELATED QUERIES"), gr.HTML()], css=css_code)
-        tab2 = gr.Interface(fn=self.qa_infer, inputs=[dropdown], allow_flagging='never', outputs=[gr.Textbox(label="SOLUTION"), gr.Textbox(label="RELATED QUERIES"), gr.HTML()], css=css_code)#, title="FAQs") 
+        # Define Gradio interface
+        interface = gr.Interface(
+            fn=doc_retrieval_gen.qa_infer_gradio,
+            inputs=[gr.Textbox(label="QUERY", placeholder="Enter your query here")],
+            allow_flagging='never',
+            examples=EXAMPLES,
+            cache_examples=False,
+            outputs=gr.Textbox(label="SOLUTION"),
+            css=css_code
+        )
 
+        # Launch Gradio interface
+        interface.launch(debug=True)
 
-        # # Add dummy outputs to each interface
-        # tab1.outputs = dummy_outputs
-        # tab2.outputs = dummy_outputs
-        
-        gr.TabbedInterface([tab1, tab2],["Textbox Input", "FAQs"],title="TI E2E FORUM",css=css_code).launch(debug=True)
-
-# Instantiate and launch the chatbot
-chatbot = Chatbot()
-chatbot.launch_interface()
-
-"""Single Tab Input Inference"""
-
-    # def launch_interface(self):
-    #     css_code = """
-    #         .gradio-container {
-    #             background-color: #daccdb;
-    #         }
-
-    #         /* Button styling for all buttons */
-    #         button {
-    #             background-color: #927fc7; /* Default color for all other buttons */
-    #             color: black;
-    #             border: 1px solid black;
-    #             padding: 10px;
-    #             margin-right: 10px;
-    #             font-size: 16px; /* Increase font size */
-    #             font-weight: bold; /* Make text bold */
-    #         }
-
-    #         """
-    #     EXAMPLES = ["TDA4 product planning and datasheet release progress? ", 
-    #                 "I'm using Code Composer Studio 5.4.0.00091 and enabled FPv4SPD16 floating point support for CortexM4 in TDA2. However, after building the project, the .asm file shows --float_support=vfplib instead of FPv4SPD16. Why is this happening?", 
-    #                 "Master core in TDA2XX is a15 and in TDA3XX it is m4,so we have to shift all modules that are being used by a15 in TDA2XX to m4 in TDA3xx."]
-
-    #     demo = gr.Interface(fn=self.qa_infer, inputs=[gr.Textbox(label="QUERY", placeholder ="Enter your query here")], allow_flagging='never', examples=EXAMPLES, cache_examples=False, outputs=[gr.Textbox(label="SOLUTION"), gr.Textbox(label="RELATED QUERIES"), gr.HTML()], css=css_code)
-    #     demo.launch()
+    # Launch the interface
+    launch_interface()