adding files

app.py

@@ -0,0 +1,80 @@
+from torch import cuda , bfloat16 
+import os
+import transformers 
+from langchain.chains import RetrievalQA 
+from langchain.llms import HuggingFacePipeline 
+from langchain.document_loaders import PyPDFLoader 
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings import HuggingFaceEmbeddings 
+from langchain.vectorstores import FAISS 
+import gradio as gr
+
+os.environ['HUGGINGFACEHUB_API_TOKEN'] = os.environ["HF_KEY"]
+os.environ['HF_TOKEN']  =  os.environ["HF_KEY"]
+
+
+model_id = 'meta-llama/Llama-2-7b-chat-hf'
+
+device= f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'
+
+
+bnb_config = transformers.BitsAndBytesConfig(
+    load_in_4bit = True,
+    bnb_4bit_quant_type='nf4',
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_compute_dtype=bfloat16
+)
+
+
+model_config = transformers.AutoConfig.from_pretrained(
+    model_id,    
+)
+
+
+model = transformers.AutoModelForCausalLM.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+    config = model_config,
+    quantization_config = bnb_config,
+    device_map='auto',
+    
+)
+
+
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+    model_id,
+)
+
+model.eval() 
+
+print(f"Model loaded on {device}")
+generate_text = transformers.pipeline(
+    model = model,
+    tokenizer = tokenizer,
+    return_full_text = True,
+    task='text-generation',    
+    temperature = 0.1,
+    max_new_tokens=512,
+    repetition_penalty=1.1
+                                      
+)
+llm = HuggingFacePipeline(pipeline=generate_text)
+# loader = PyPDFLoader("/content/CELEX%3A32023R1115%3AEN%3ATXT.pdf")
+# pdf_documents = loader.load()
+# text_splitter  = RecursiveCharacterTextSplitter(chunk_size=1024,chunk_overlap=100)
+# pdf_document_chunks = text_splitter.split_documents(pdf_documents)
+model_name = "sentence-transformers/all-mpnet-base-v2"
+# model_kwargs = {'device':'cpu'}
+embeddings = HuggingFaceEmbeddings(model_name = model_name)#,model_kwargs=model_kwargs)
+vectorstore = FAISS.load_local("faiss_index",embeddings)
+retrievalQA = RetrievalQA.from_chain_type(llm,
+                                          chain_type='stuff',
+                                          retriever = vectorstore.as_retriever(),return_source_documents=True)
+
+print("setup complete lets start answering questions")
+def question_answer(input):
+    response = retrievalQA.invoke(input)
+    return response['result'],response['source_documents'][0].page_content
+
+iface = gr.Interface(fn = question_answer,inputs='text',outputs=['text','text'])
+iface.launch()

requirements.txt

@@ -0,0 +1,10 @@
+accelerate==0.21.0 
+transformers==4.31.0 
+tokenizers==0.13.3
+bitsandbytes==0.40.0 
+einops==0.6.1
+xformers==0.0.22.post7
+langchain==0.1.4
+faiss-gpu==1.7.1.post3
+sentence_transformers
+gradio