Upload 5 files

src/gradio_chatbot.py

@@ -0,0 +1,81 @@
+import gradio as gr
+from model_utils import *
+
+
+with gr.Blocks(gr.themes.Soft(primary_hue=gr.themes.colors.slate, secondary_hue=gr.themes.colors.purple)) as demo:
+    gr.Markdown('''# Retrieval Augmented Generation \n
+                     RAG (Retrieval-Augmented Generation) addresses the data freshness problem in Large Language Models (LLMs) like Llama-2, which lack awareness of recent events. LLMs perceive the world only through their training data, leading to challenges when needing up-to-date information or specific datasets. To tackle this, retrieval augmentation is employed, enabling relevant external knowledge from a knowledge base to be incorporated into LLM responses.
+                     RAG involves creating a knowledge base containing two types of knowledge: parametric knowledge from LLM training and source knowledge from external input. Data for the knowledge base is derived from datasets relevant to the use case, which are then processed into smaller chunks to enhance relevance and efficiency. Token embeddings, generated using models like RoBERTa, are crucial for retrieving context and meaning from the knowledge base.
+                     A vector database could be used to manage and search through the embeddings efficiently. The LangChain library facilitates interactions with the knowledge base, allowing LLMs to generate responses based on retrieved information. Generative Question Answering (GQA) or Retrieval Augmented Generation (RAG) techniques instruct the LLM to craft answers using knowledge base content. To enhance trust, answers can be accompanied by citations indicating the information source.
+                     RAG leverages a combination of external knowledge and LLM capabilities to provide accurate, up-to-date, and well-grounded responses. This approach is gaining traction in products such as AI search engines and conversational agents, highlighting the synergy between LLMs and robust knowledge bases.
+                ''')
+    with gr.Row():
+
+        with gr.Column(scale=0.5, variant = 'panel'):
+            gr.Markdown("## Upload Document & Select the Embedding Model")
+            file = gr.File(type="file")
+            with gr.Row(equal_height=True):
+                
+                with gr.Column(scale=0.5, variant = 'panel'):
+                    embedding_model = gr.Dropdown(choices= ["all-roberta-large-v1_1024d", "all-mpnet-base-v2_768d"],
+                                    value="all-roberta-large-v1_1024d",
+                                    label= "Select the embedding model")
+
+                with gr.Column(scale=0.5, variant='compact'):
+                    vector_index_btn = gr.Button('Create vector store', variant='primary',scale=1)
+                    vector_index_msg_out = gr.Textbox(show_label=False, lines=1,scale=1, placeholder="Creating vectore store ...")
+
+            instruction = gr.Textbox(label="System instruction", lines=3, value="Use the following pieces of context to answer the question at the end by. Generate the answer based on the given context only.If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive.")
+            reset_inst_btn = gr.Button('Reset',variant='primary', size = 'sm')
+
+            with gr.Accordion(label="Text generation tuning parameters"):
+                temperature = gr.Slider(label="temperature", minimum=0.1, maximum=1, value=0.1, step=0.05)
+                max_new_tokens = gr.Slider(label="max_new_tokens", minimum=1, maximum=2048, value=512, step=1)
+                repetition_penalty = gr.Slider(label="repetition_penalty", minimum=0, maximum=2, value=1.1, step=0.1)
+                top_k= gr.Slider(label="top_k", minimum=1, maximum=1000, value=10, step=1)
+                top_p=gr.Slider(label="top_p", minimum=0, maximum=1, value=0.95, step=0.05)
+                k_context=gr.Slider(label="k_context", minimum=1, maximum=15, value=5, step=1)
+
+            vector_index_btn.click(upload_and_create_vector_store,[file,embedding_model],vector_index_msg_out)
+            reset_inst_btn.click(reset_sys_instruction,instruction,instruction)
+
+        with gr.Column(scale=0.5, variant = 'panel'):
+            gr.Markdown("## Select the Generation Model")
+
+            with gr.Row(equal_height=True):
+
+                with gr.Column(scale=0.5):
+                    llm = gr.Dropdown(choices= ["Llamav2-7B-Chat", "Falcon-7B-Instruct"], value="Llamav2-7B-Chat", label="Select the LLM")
+                    hf_token = gr.Textbox(label='Enter your valid HF token_id', type = "password")
+
+                with gr.Column(scale=0.5):
+                    model_load_btn = gr.Button('Load model', variant='primary',scale=2)
+                    load_success_msg = gr.Textbox(show_label=False,lines=1, placeholder="Model loading ...")
+            chatbot = gr.Chatbot([], elem_id="chatbot",
+                                label='Chatbox', height=725, )
+
+            txt = gr.Textbox(label= "Question",lines=2,placeholder="Enter your question and press shift+enter ")
+
+            with gr.Row():
+
+                with gr.Column(scale=0.5):
+                    submit_btn = gr.Button('Submit',variant='primary', size = 'sm')
+
+                with gr.Column(scale=0.5):
+                    clear_btn = gr.Button('Clear',variant='stop',size = 'sm')
+
+            model_load_btn.click(load_models, [hf_token,embedding_model,llm], load_success_msg, api_name="load_models")
+
+            txt.submit(add_text, [chatbot, txt], [chatbot, txt]).then(
+                bot, [chatbot,instruction,temperature,max_new_tokens,repetition_penalty,top_k,top_p,k_context], chatbot)
+            submit_btn.click(add_text, [chatbot, txt], [chatbot, txt]).then(
+                bot, [chatbot,instruction,temperature, max_new_tokens,repetition_penalty,top_k,top_p,k_context], chatbot).then(
+                    clear_cuda_cache, None, None
+                )
+
+            clear_btn.click(lambda: None, None, chatbot, queue=False)
+
+
+if __name__ == '__main__':
+    demo.queue(concurrency_count=3)
+    demo.launch(debug=True, share=True)

src/model_setup.py

@@ -0,0 +1,32 @@
+import os,gc,shutil
+from util.conversation_rag import Conversation_RAG
+from util.index import *
+import torch
+
+class ModelSetup:
+    def __init__(self, hf_token, embedding_model, llm):
+
+        self.hf_token = hf_token
+        self.embedding_model = embedding_model
+        self.llm = llm
+
+    def setup(self):
+
+        if self.embedding_model == "all-roberta-large-v1_1024d":
+            embedding_model_repo_id = "sentence-transformers/all-roberta-large-v1"
+        elif self.embedding_model == "all-mpnet-base-v2_768d":
+            embedding_model_repo_id = "sentence-transformers/all-mpnet-base-v2"
+
+
+        if self.llm == "Llamav2-7B-Chat":
+            llm_repo_id = "meta-llama/Llama-2-7b-chat-hf"
+        elif self.llm == "Falcon-7B-Instruct":
+            llm_repo_id = "tiiuae/falcon-7b-instruct"
+
+
+        conv_rag = Conversation_RAG(self.hf_token,
+                                    embedding_model_repo_id,
+                                    llm_repo_id)
+
+        self.model, self.tokenizer, self.vectordb = conv_rag.load_model_and_tokenizer()
+        return "Model Setup Complete"

src/model_utils.py

@@ -0,0 +1,99 @@
+import os,gc,shutil
+import gradio as gr
+from util.conversation_rag import Conversation_RAG
+from util.index import *
+import torch
+from model_setup import ModelSetup
+
+
+def load_models(hf_token,embedding_model,llm):
+
+    global model_setup
+    model_setup = ModelSetup(hf_token, embedding_model, llm)
+    success_prompt = model_setup.setup()
+    return success_prompt
+
+
+def upload_and_create_vector_store(file,embedding_model):
+    
+    # Save the uploaded file to a permanent location
+    file_path = file.name
+    split_file_name = file_path.split("/")
+    file_name = split_file_name[-1]
+
+    current_folder = os.path.dirname(os.path.abspath(__file__))
+    root_folder = os.path.dirname(current_folder)
+    data_folder = os.path.join(root_folder, "data")
+    permanent_file_path = os.path.join(data_folder, file_name)
+    shutil.copy(file.name, permanent_file_path)
+
+    # Access the path of the saved file
+    print(f"File saved to: {permanent_file_path}")
+
+    if embedding_model == "all-roberta-large-v1_1024d":
+        embedding_model_repo_id = "sentence-transformers/all-roberta-large-v1"
+    elif embedding_model == "all-mpnet-base-v2_768d":
+        embedding_model_repo_id = "sentence-transformers/all-mpnet-base-v2"
+
+    index_success_msg = create_vector_store_index(permanent_file_path,embedding_model_repo_id)
+    return index_success_msg
+
+def get_chat_history(inputs):
+
+    res = []
+    for human, ai in inputs:
+        res.append(f"Human:{human}\nAssistant:{ai}")
+    return "\n".join(res)
+
+def add_text(history, text):
+
+    history = history + [[text, None]]
+    return history, ""
+
+conv_qa = Conversation_RAG()
+def bot(history,
+        instruction="Use the following pieces of context to answer the question at the end. Generate the answer based on the given context only if you find the answer in the context. If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive.",
+        temperature=0.1,
+        max_new_tokens=512,
+        repetition_penalty=1.1,
+        top_k=10,
+        top_p=0.95,
+        k_context=5,
+        num_return_sequences=1,
+        ):
+
+    qa = conv_qa.create_conversation(model_setup.model,
+                             model_setup.tokenizer,
+                             model_setup.vectordb,
+                             max_new_tokens=max_new_tokens,
+                             temperature=temperature,
+                             repetition_penalty=repetition_penalty,
+                             top_k=top_k,
+                             top_p=top_p,
+                             k_context=k_context,
+                             num_return_sequences=num_return_sequences,
+                             instruction=instruction
+
+    )
+
+    chat_history_formatted = get_chat_history(history[:-1])
+    res = qa(
+        {
+            'question': history[-1][0],
+            'chat_history': chat_history_formatted
+        }
+    )
+    
+    history[-1][1] = res['answer']
+    return history
+
+def reset_sys_instruction(instruction):
+
+    default_inst = "Use the following pieces of context to answer the question at the end. Generate the answer based on the given context only if you find the answer in the context. If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive."
+    return default_inst
+
+def clear_cuda_cache():
+
+    torch.cuda.empty_cache()
+    gc.collect()
+    return None

src/util/conversation_rag.py

@@ -0,0 +1,85 @@
+from torch import cuda, bfloat16
+import transformers
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains import ConversationalRetrievalChain
+from langchain.memory import ConversationBufferMemory
+from langchain.llms import HuggingFacePipeline
+from huggingface_hub import login
+from langchain.prompts import PromptTemplate
+
+
+class Conversation_RAG:
+    def __init__(self, hf_token = "", embedding_model_repo_id="sentence-transformers/all-roberta-large-v1",
+                 llm_repo_id='meta-llama/Llama-2-7b-chat-hf'):
+        
+        self.hf_token = hf_token
+        self.embedding_model_repo_id = embedding_model_repo_id
+        self.llm_repo_id = llm_repo_id
+
+    def load_model_and_tokenizer(self):
+
+        embedding_model = HuggingFaceEmbeddings(model_name=self.embedding_model_repo_id)
+        vectordb = FAISS.load_local("./db/faiss_index", embedding_model)
+
+        login(token=self.hf_token)
+
+        device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'
+
+        bnb_config = transformers.BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type='nf4',
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_compute_dtype=bfloat16
+        )
+
+        model = transformers.AutoModelForCausalLM.from_pretrained(
+            self.llm_repo_id,
+            trust_remote_code=True,
+            quantization_config=bnb_config,
+            load_in_8bit=True,
+            device_map='auto'
+        )
+        model.eval()
+
+        tokenizer = transformers.AutoTokenizer.from_pretrained(self.llm_repo_id)
+        return model, tokenizer, vectordb
+
+    def create_conversation(self, model, tokenizer, vectordb, max_new_tokens=512, temperature=0.1, repetition_penalty=1.1, top_k=10, top_p=0.95, k_context=5,
+                            num_return_sequences=1, instruction="Use the following pieces of context to answer the question at the end by. Generate the answer based on the given context only. If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive."):
+
+        generate_text = transformers.pipeline(
+            model=model,
+            tokenizer=tokenizer,
+            return_full_text=True,  # langchain expects the full text
+            task='text-generation',
+            temperature=temperature,  # 'randomness' of outputs, 0.0 is the min and 1.0 the max
+            max_new_tokens=max_new_tokens,  # mex number of tokens to generate in the output
+            repetition_penalty=repetition_penalty,  # without this output begins repeating
+            top_k=top_k,
+            top_p=top_p,
+            num_return_sequences=num_return_sequences,
+        )
+
+        llm = HuggingFacePipeline(pipeline=generate_text)
+
+        system_instruction = f"User: {instruction}\n"
+        template = system_instruction + """
+        context:\n
+        {context}\n
+        Question: {question}\n
+        Assistant:
+        """
+
+        QCA_PROMPT = PromptTemplate(input_variables=["context", "question"], template=template)
+
+        qa = ConversationalRetrievalChain.from_llm(
+            llm=llm,
+            chain_type='stuff',
+            retriever=vectordb.as_retriever(search_kwargs={"k": k_context}),
+            combine_docs_chain_kwargs={"prompt": QCA_PROMPT},
+            get_chat_history=lambda h: h,
+            verbose=True
+        )
+        return qa
+

src/util/index.py

@@ -0,0 +1,41 @@
+import os
+import numpy as np
+import pickle
+
+from langchain.vectorstores import FAISS, Chroma, DocArrayInMemorySearch
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+def create_vector_store_index(file_path, embedding_model_repo_id="sentence-transformers/all-roberta-large-v1"):
+
+    file_path_split = file_path.split(".")
+    file_type = file_path_split[-1].rstrip('/')
+
+    if file_type == 'csv':
+        print(file_path)
+        loader = CSVLoader(file_path=file_path)
+        documents = loader.load()
+    
+    elif file_type == 'pdf':
+        loader = PyPDFLoader(file_path)
+        pages = loader.load()
+        
+        text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size = 1024,
+        chunk_overlap = 128,)
+
+        documents = text_splitter.split_documents(pages)
+
+    
+    embedding_model = HuggingFaceEmbeddings(
+        model_name=embedding_model_repo_id
+        )
+    
+    vectordb = FAISS.from_documents(documents, embedding_model)
+    file_output = "./db/faiss_index"
+    vectordb.save_local(file_output)
+
+    return "Vector store index is created."