application

gradio_RAG-main/README.md

@@ -0,0 +1,41 @@
+# Retrieval Augmented Generation
+
+This repository contains code and resources related to Retrieval Augmented Generation (RAG), a technique designed to address the data freshness problem in Large Language Models (LLMs) like Llama-2. LLMs often lack awareness of recent events and up-to-date information. RAG incorporates external knowledge from a knowledge base into LLM responses, enabling accurate and well-grounded responses.
+
+## Repository Contents
+
+- `src`: Contains the source code for implementing the RAG technique and interactions with the knowledge base.
+- `data`: Stores datasets and relevant resources for building the knowledge base.
+- `db`: To manage and store token embeddings or vector representations for knowledge base searches.
+- `requirements.txt`: Required Python packages to run the code in this repository.
+
+## About RAG (Retrieval Augmented Generation)
+
+RAG is a novel approach combining Large Language Models (LLMs) capabilities with external knowledge bases to enhance the quality and freshness of generated responses. It addresses the challenge of outdated information by retrieving contextually relevant knowledge from external sources and incorporating it into LLM-generated content.
+
+## About Gradio
+
+[Gradio](https://www.gradio.app) is a Python library that helps you quickly create UIs for your machine learning models. It allows you to quickly deploy models and make them accessible through a user-friendly interface without extensive frontend development.
+
+A Gradio app is launched when `gradio_chatbot.py` code is run. It contains modifiable elements such as the Embedding model, Generation model, editable system prompt, and tunable parameters of the chosen LLM.
+
+### Steps
+
+To use the code in this repository, follow these steps:
+
+1. Clone the repository to your local machine.
+2. Navigate to the repository directory using the command line.
+3. Install the required packages using the following command:
+
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+4. Run the chatbot application using the command:
+
+   ```bash
+   python src/gradio_chatbot.py
+   ```
+
+5. Once the Gradio app is up, upload a document (pdf or csv), choose the models (embedding and generation), adjust the tunable parameters, fiddle with the system prompt, and ask anything you need!
+

gradio_RAG-main/db/faiss_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bdc86b65b84ca0688c5a3c59868d6e196e56810fda839d7518fc898e616ba3e
+size 55006

gradio_RAG-main/requirements.txt

@@ -0,0 +1,9 @@
+gradio
+transformers
+accelerate
+Xformers
+langchain
+bitsandbytes
+sentence_transformers
+faiss-cpu
+PyPDF

gradio_RAG-main/src/gradio_chatbot.py

@@ -0,0 +1,81 @@
+import gradio as gr
+from model_utils import *
+
+
+with gr.Blocks(gr.themes.Soft(primary_hue=gr.themes.colors.slate, secondary_hue=gr.themes.colors.purple)) as demo:
+    gr.Markdown('''# Retrieval Augmented Generation \n
+                     RAG (Retrieval-Augmented Generation) addresses the data freshness problem in Large Language Models (LLMs) like Llama-2, which lack awareness of recent events. LLMs perceive the world only through their training data, leading to challenges when needing up-to-date information or specific datasets. To tackle this, retrieval augmentation is employed, enabling relevant external knowledge from a knowledge base to be incorporated into LLM responses.
+                     RAG involves creating a knowledge base containing two types of knowledge: parametric knowledge from LLM training and source knowledge from external input. Data for the knowledge base is derived from datasets relevant to the use case, which are then processed into smaller chunks to enhance relevance and efficiency. Token embeddings, generated using models like RoBERTa, are crucial for retrieving context and meaning from the knowledge base.
+                     A vector database could be used to manage and search through the embeddings efficiently. The LangChain library facilitates interactions with the knowledge base, allowing LLMs to generate responses based on retrieved information. Generative Question Answering (GQA) or Retrieval Augmented Generation (RAG) techniques instruct the LLM to craft answers using knowledge base content. To enhance trust, answers can be accompanied by citations indicating the information source.
+                     RAG leverages a combination of external knowledge and LLM capabilities to provide accurate, up-to-date, and well-grounded responses. This approach is gaining traction in products such as AI search engines and conversational agents, highlighting the synergy between LLMs and robust knowledge bases.
+                ''')
+    with gr.Row():
+
+        with gr.Column(scale=0.5, variant = 'panel'):
+            gr.Markdown("## Upload Document & Select the Embedding Model")
+            file = gr.File(type="file")
+            with gr.Row(equal_height=True):
+                
+                with gr.Column(scale=0.5, variant = 'panel'):
+                    embedding_model = gr.Dropdown(choices= ["all-roberta-large-v1_1024d", "all-mpnet-base-v2_768d"],
+                                    value="all-roberta-large-v1_1024d",
+                                    label= "Select the embedding model")
+
+                with gr.Column(scale=0.5, variant='compact'):
+                    vector_index_btn = gr.Button('Create vector store', variant='primary',scale=1)
+                    vector_index_msg_out = gr.Textbox(show_label=False, lines=1,scale=1, placeholder="Creating vectore store ...")
+
+            instruction = gr.Textbox(label="System instruction", lines=3, value="Use the following pieces of context to answer the question at the end by. Generate the answer based on the given context only.If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive.")
+            reset_inst_btn = gr.Button('Reset',variant='primary', size = 'sm')
+
+            with gr.Accordion(label="Text generation tuning parameters"):
+                temperature = gr.Slider(label="temperature", minimum=0.1, maximum=1, value=0.1, step=0.05)
+                max_new_tokens = gr.Slider(label="max_new_tokens", minimum=1, maximum=2048, value=512, step=1)
+                repetition_penalty = gr.Slider(label="repetition_penalty", minimum=0, maximum=2, value=1.1, step=0.1)
+                top_k= gr.Slider(label="top_k", minimum=1, maximum=1000, value=10, step=1)
+                top_p=gr.Slider(label="top_p", minimum=0, maximum=1, value=0.95, step=0.05)
+                k_context=gr.Slider(label="k_context", minimum=1, maximum=15, value=5, step=1)
+
+            vector_index_btn.click(upload_and_create_vector_store,[file,embedding_model],vector_index_msg_out)
+            reset_inst_btn.click(reset_sys_instruction,instruction,instruction)
+
+        with gr.Column(scale=0.5, variant = 'panel'):
+            gr.Markdown("## Select the Generation Model")
+
+            with gr.Row(equal_height=True):
+
+                with gr.Column(scale=0.5):
+                    llm = gr.Dropdown(choices= ["Llamav2-7B-Chat", "Falcon-7B-Instruct"], value="Llamav2-7B-Chat", label="Select the LLM")
+                    hf_token = gr.Textbox(label='Enter your valid HF token_id', type = "password")
+
+                with gr.Column(scale=0.5):
+                    model_load_btn = gr.Button('Load model', variant='primary',scale=2)
+                    load_success_msg = gr.Textbox(show_label=False,lines=1, placeholder="Model loading ...")
+            chatbot = gr.Chatbot([], elem_id="chatbot",
+                                label='Chatbox', height=725, )
+
+            txt = gr.Textbox(label= "Question",lines=2,placeholder="Enter your question and press shift+enter ")
+
+            with gr.Row():
+
+                with gr.Column(scale=0.5):
+                    submit_btn = gr.Button('Submit',variant='primary', size = 'sm')
+
+                with gr.Column(scale=0.5):
+                    clear_btn = gr.Button('Clear',variant='stop',size = 'sm')
+
+            model_load_btn.click(load_models, [hf_token,embedding_model,llm], load_success_msg, api_name="load_models")
+
+            txt.submit(add_text, [chatbot, txt], [chatbot, txt]).then(
+                bot, [chatbot,instruction,temperature,max_new_tokens,repetition_penalty,top_k,top_p,k_context], chatbot)
+            submit_btn.click(add_text, [chatbot, txt], [chatbot, txt]).then(
+                bot, [chatbot,instruction,temperature, max_new_tokens,repetition_penalty,top_k,top_p,k_context], chatbot).then(
+                    clear_cuda_cache, None, None
+                )
+
+            clear_btn.click(lambda: None, None, chatbot, queue=False)
+
+
+if __name__ == '__main__':
+    demo.queue(concurrency_count=3)
+    demo.launch(debug=True, share=True)

gradio_RAG-main/src/model_setup.py

@@ -0,0 +1,32 @@
+import os,gc,shutil
+from util.conversation_rag import Conversation_RAG
+from util.index import *
+import torch
+
+class ModelSetup:
+    def __init__(self, hf_token, embedding_model, llm):
+
+        self.hf_token = hf_token
+        self.embedding_model = embedding_model
+        self.llm = llm
+
+    def setup(self):
+
+        if self.embedding_model == "all-roberta-large-v1_1024d":
+            embedding_model_repo_id = "sentence-transformers/all-roberta-large-v1"
+        elif self.embedding_model == "all-mpnet-base-v2_768d":
+            embedding_model_repo_id = "sentence-transformers/all-mpnet-base-v2"
+
+
+        if self.llm == "Llamav2-7B-Chat":
+            llm_repo_id = "meta-llama/Llama-2-7b-chat-hf"
+        elif self.llm == "Falcon-7B-Instruct":
+            llm_repo_id = "tiiuae/falcon-7b-instruct"
+
+
+        conv_rag = Conversation_RAG(self.hf_token,
+                                    embedding_model_repo_id,
+                                    llm_repo_id)
+
+        self.model, self.tokenizer, self.vectordb = conv_rag.load_model_and_tokenizer()
+        return "Model Setup Complete"

gradio_RAG-main/src/model_utils.py

@@ -0,0 +1,99 @@
+import os,gc,shutil
+import gradio as gr
+from util.conversation_rag import Conversation_RAG
+from util.index import *
+import torch
+from model_setup import ModelSetup
+
+
+def load_models(hf_token,embedding_model,llm):
+
+    global model_setup
+    model_setup = ModelSetup(hf_token, embedding_model, llm)
+    success_prompt = model_setup.setup()
+    return success_prompt
+
+
+def upload_and_create_vector_store(file,embedding_model):
+    
+    # Save the uploaded file to a permanent location
+    file_path = file.name
+    split_file_name = file_path.split("/")
+    file_name = split_file_name[-1]
+
+    current_folder = os.path.dirname(os.path.abspath(__file__))
+    root_folder = os.path.dirname(current_folder)
+    data_folder = os.path.join(root_folder, "data")
+    permanent_file_path = os.path.join(data_folder, file_name)
+    shutil.copy(file.name, permanent_file_path)
+
+    # Access the path of the saved file
+    print(f"File saved to: {permanent_file_path}")
+
+    if embedding_model == "all-roberta-large-v1_1024d":
+        embedding_model_repo_id = "sentence-transformers/all-roberta-large-v1"
+    elif embedding_model == "all-mpnet-base-v2_768d":
+        embedding_model_repo_id = "sentence-transformers/all-mpnet-base-v2"
+
+    index_success_msg = create_vector_store_index(permanent_file_path,embedding_model_repo_id)
+    return index_success_msg
+
+def get_chat_history(inputs):
+
+    res = []
+    for human, ai in inputs:
+        res.append(f"Human:{human}\nAssistant:{ai}")
+    return "\n".join(res)
+
+def add_text(history, text):
+
+    history = history + [[text, None]]
+    return history, ""
+
+conv_qa = Conversation_RAG()
+def bot(history,
+        instruction="Use the following pieces of context to answer the question at the end. Generate the answer based on the given context only if you find the answer in the context. If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive.",
+        temperature=0.1,
+        max_new_tokens=512,
+        repetition_penalty=1.1,
+        top_k=10,
+        top_p=0.95,
+        k_context=5,
+        num_return_sequences=1,
+        ):
+
+    qa = conv_qa.create_conversation(model_setup.model,
+                             model_setup.tokenizer,
+                             model_setup.vectordb,
+                             max_new_tokens=max_new_tokens,
+                             temperature=temperature,
+                             repetition_penalty=repetition_penalty,
+                             top_k=top_k,
+                             top_p=top_p,
+                             k_context=k_context,
+                             num_return_sequences=num_return_sequences,
+                             instruction=instruction
+
+    )
+
+    chat_history_formatted = get_chat_history(history[:-1])
+    res = qa(
+        {
+            'question': history[-1][0],
+            'chat_history': chat_history_formatted
+        }
+    )
+    
+    history[-1][1] = res['answer']
+    return history
+
+def reset_sys_instruction(instruction):
+
+    default_inst = "Use the following pieces of context to answer the question at the end. Generate the answer based on the given context only if you find the answer in the context. If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive."
+    return default_inst
+
+def clear_cuda_cache():
+
+    torch.cuda.empty_cache()
+    gc.collect()
+    return None

gradio_RAG-main/src/util/conversation_rag.py

@@ -0,0 +1,85 @@
+from torch import cuda, bfloat16
+import transformers
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains import ConversationalRetrievalChain
+from langchain.memory import ConversationBufferMemory
+from langchain.llms import HuggingFacePipeline
+from huggingface_hub import login
+from langchain.prompts import PromptTemplate
+
+
+class Conversation_RAG:
+    def __init__(self, hf_token = "", embedding_model_repo_id="sentence-transformers/all-roberta-large-v1",
+                 llm_repo_id='meta-llama/Llama-2-7b-chat-hf'):
+        
+        self.hf_token = hf_token
+        self.embedding_model_repo_id = embedding_model_repo_id
+        self.llm_repo_id = llm_repo_id
+
+    def load_model_and_tokenizer(self):
+
+        embedding_model = HuggingFaceEmbeddings(model_name=self.embedding_model_repo_id)
+        vectordb = FAISS.load_local("./db/faiss_index", embedding_model)
+
+        login(token=self.hf_token)
+
+        device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'
+
+        bnb_config = transformers.BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type='nf4',
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_compute_dtype=bfloat16
+        )
+
+        model = transformers.AutoModelForCausalLM.from_pretrained(
+            self.llm_repo_id,
+            trust_remote_code=True,
+            quantization_config=bnb_config,
+            load_in_8bit=True,
+            device_map='auto'
+        )
+        model.eval()
+
+        tokenizer = transformers.AutoTokenizer.from_pretrained(self.llm_repo_id)
+        return model, tokenizer, vectordb
+
+    def create_conversation(self, model, tokenizer, vectordb, max_new_tokens=512, temperature=0.1, repetition_penalty=1.1, top_k=10, top_p=0.95, k_context=5,
+                            num_return_sequences=1, instruction="Use the following pieces of context to answer the question at the end by. Generate the answer based on the given context only. If you do not find any information related to the question in the given context, just say that you don't know, don't try to make up an answer. Keep your answer expressive."):
+
+        generate_text = transformers.pipeline(
+            model=model,
+            tokenizer=tokenizer,
+            return_full_text=True,  # langchain expects the full text
+            task='text-generation',
+            temperature=temperature,  # 'randomness' of outputs, 0.0 is the min and 1.0 the max
+            max_new_tokens=max_new_tokens,  # mex number of tokens to generate in the output
+            repetition_penalty=repetition_penalty,  # without this output begins repeating
+            top_k=top_k,
+            top_p=top_p,
+            num_return_sequences=num_return_sequences,
+        )
+
+        llm = HuggingFacePipeline(pipeline=generate_text)
+
+        system_instruction = f"User: {instruction}\n"
+        template = system_instruction + """
+        context:\n
+        {context}\n
+        Question: {question}\n
+        Assistant:
+        """
+
+        QCA_PROMPT = PromptTemplate(input_variables=["context", "question"], template=template)
+
+        qa = ConversationalRetrievalChain.from_llm(
+            llm=llm,
+            chain_type='stuff',
+            retriever=vectordb.as_retriever(search_kwargs={"k": k_context}),
+            combine_docs_chain_kwargs={"prompt": QCA_PROMPT},
+            get_chat_history=lambda h: h,
+            verbose=True
+        )
+        return qa
+

gradio_RAG-main/src/util/index.py

@@ -0,0 +1,41 @@
+import os
+import numpy as np
+import pickle
+
+from langchain.vectorstores import FAISS, Chroma, DocArrayInMemorySearch
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+def create_vector_store_index(file_path, embedding_model_repo_id="sentence-transformers/all-roberta-large-v1"):
+
+    file_path_split = file_path.split(".")
+    file_type = file_path_split[-1].rstrip('/')
+
+    if file_type == 'csv':
+        print(file_path)
+        loader = CSVLoader(file_path=file_path)
+        documents = loader.load()
+    
+    elif file_type == 'pdf':
+        loader = PyPDFLoader(file_path)
+        pages = loader.load()
+        
+        text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size = 1024,
+        chunk_overlap = 128,)
+
+        documents = text_splitter.split_documents(pages)
+
+    
+    embedding_model = HuggingFaceEmbeddings(
+        model_name=embedding_model_repo_id
+        )
+    
+    vectordb = FAISS.from_documents(documents, embedding_model)
+    file_output = "./db/faiss_index"
+    vectordb.save_local(file_output)
+
+    return "Vector store index is created."