Upload 34 files

.gitattributes

@@ -33,3 +33,33 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+uploads/Adani_Ports.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Asian_Paints.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Axis_Bank.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Bajaj_Finance.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Bajaj_Finserv.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Bharti_Airtel.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/HCL_Tech.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/HDFC_Bank.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Hindustan_Unilever.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/ICICI_Bank.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/IndusInd_Bank.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Infosys.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/ITC.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/JSW_Steel.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Kotak_Bank.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/L&T.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Mahindra&Mahindra.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Maruti_Suzuki.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Nestle.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/NTPC.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Powergrid.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Reliance.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/SBI_Bank.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Sun_Pharma.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Tata_Motors.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Tata_Steel.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/TCS.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Tech_Mahindra.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Titan.pdf filter=lfs diff=lfs merge=lfs -text
+uploads/Ultratech.pdf filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,129 @@
+import gradio as gr 
+import os 
+from tqdm import tqdm
+
+from unstructured.partition.pdf import partition_pdf
+from langchain.schema.document import Document
+import google.generativeai as genai
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+import shutil
+
+from wasabi import msg
+
+from PIL import Image
+
+genai.configure(api_key = 'AIzaSyB342Fh-nkRaO38BshbyI4-s0T9orVpsMw')
+model = genai.GenerativeModel('gemini-1.5-flash')
+
+files = os.listdir('uploads')
+files = [f'uploads/{file}' for file in files]
+
+documents = []
+
+for file in tqdm(files , total = len(files) , leave = False) : 
+
+    elements = partition_pdf(
+        filename = file , 
+        extract_images_in_pdf = True , 
+        infer_table_structure = True , 
+        chunking_strategy = 'by_title' , 
+        max_characters = 4000 , 
+        new_after_n_chars = 3800 , 
+        combine_text_under_n_chars = 2000 , 
+        extract_image_block_output_dir = 'outputs'
+    )
+
+    for element in elements : 
+
+        element = element.to_dict()
+
+        metadata = element['metadata']
+
+        if 'text_as_html' in metadata : documents.append(
+            Document(
+                page_content = metadata['text_as_html'] , 
+                metadata = {
+                    'type' : 'text' , 
+                    'metadata' : element
+                }
+            )
+        )         
+
+        else : documents.append(
+            Document(
+                page_content = element['text'] , 
+                metadata = {
+                    'type' : 'text' , 
+                    'metadata' : element
+                }
+            )
+        )
+
+    images = os.listdir('outputs')
+    images = [f'outputs/{image}' for image in images]
+
+    for image in tqdm(images , total = len(images) , leave = False) : 
+
+        image = Image.open(image)
+
+        try : 
+
+            response = model.generate_content([
+                image , 
+                'Explain the Image'
+            ])
+
+            response = response.text
+
+        except Exception as e : msg.fail(f'----| FAIL : COULDNT CALL THE IMAGE DESCRIPTION API : {e}') ; response = 'COuldnt Call Model for this'
+
+        documents.append(
+            Document(
+                page_content = response , 
+                metadata = {
+                    'type' : 'image' , 
+                    'metadata' : {
+                        'image' : image
+                    }
+                }
+            )
+        )
+
+    shutil.rmtree('uploads')
+
+vc = FAISS.from_documents(
+    documents = documents , 
+    embedding = HuggingFaceEmbeddings(model_name = 'all-MiniLM-L6-v2')
+)
+
+def run_rag(query) : 
+
+    similar_docs = vc.similarity_search(query , k = 4)
+    context = [doc.page_content for doc in similar_docs]
+
+    prompt = f'''
+    You are a Helpfull Chatbot that helps users with their queries
+
+    - You will be provided with a query
+    - You will be provided with a context as well 
+
+    Your task is to generate a response to the query based on the context provided
+
+    Context : {context}
+
+    Query : {query}
+
+    '''
+
+    response = model.generate_content(prompt)
+
+    return response.text
+
+demo = gr.Interface(
+    fn = run_rag , 
+    inputs = 'text' , 
+    outputs = 'text'
+)
+
+demo.launch()

packages.txt

@@ -0,0 +1,2 @@
+poppler-utils 
+tesseract-ocr

requirements.txt

@@ -0,0 +1,18 @@
+unstructured 
+unstructured_inference
+unstructured_pytesseract
+pytesseract 
+pi_heif
+pdfminer.six
+langchain 
+langchain_community 
+langchain_huggingface 
+sentence-transformers 
+faiss-cpu
+groq 
+google 
+google-generativeai
+gradio
+tqdm
+Pillow
+wasabi

uploads/Adani_Ports.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e14479f84d48f530cf11f37b8b45f327b98908fe4c714c09c042f44e1b40e15d
+size 23441567

uploads/Asian_Paints.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a1375324a65d21104e91597c2a8c974f29da03c6e275c47456eb9ee3f317858
+size 26576588

uploads/Axis_Bank.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da21de351524368d87dd801a2c97b86de799bd3d6fde21c94a57686e41e8196d
+size 11263914

uploads/Bajaj_Finance.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49071340391999e6863619b33ceb8a52c5c3a6d19cf94a7f5577fd6ea2304075
+size 19813034

uploads/Bajaj_Finserv.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8743975cc9720e2db0f861da7ddc45b97c85ae5bea2202a91a9e0eb230604edf
+size 22926656

uploads/Bharti_Airtel.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab401f9504fd5fd7cf840235ecd411cf69eb82d4a853a2e6a9d70e4724801e89
+size 25376207

uploads/HCL_Tech.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf28874294660cb882e56c84021ce910fea0a1d5a150649604d051e72174d21a
+size 8340206

uploads/HDFC_Bank.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcd7ef37de26997fed3bcf2b7cfb9eb90f08d4e2617f5f3f967a1ac621c2a73e
+size 17350949

uploads/Hindustan_Unilever.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffb3d9f6bde2d6fa5882011ad4d394f1d6ddbdfcd3d6080617d652a90fa10595
+size 21616344

uploads/ICICI_Bank.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a62858135454fc5f6e09b018102cba98813928d2b2e72023e86194c0087cd09
+size 11777065

uploads/ITC.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cef9cf09cefa71094985018a88b936733ff2c3a02b7cc6cd772fdeb8b75a9c97
+size 24199409

uploads/IndusInd_Bank.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24102ada097cfc42f46cc2fb4d3f520fae76e01749075491cca4e6d0be91ea93
+size 19735093

uploads/Infosys.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0a9bb9e802aff5f09733b8c78c88e9878732ac46e0fb29754c6da87ad47326a
+size 11441269

uploads/JSW_Steel.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5946803f37a1c099be7b58fb4a8746f57ebd475ef7b673994e93746ffc62b508
+size 17148166

uploads/Kotak_Bank.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea21e25405bc8c9badb4085b42eade96559f4b915a2f383b4fee33a151d47602
+size 18338231

uploads/L&T.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a86677b95602b7ebafcd16295deafbd7a2cba4a90c502aed37f05dee9dafcf8a
+size 18910726

uploads/Mahindra&Mahindra.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d46740998cf312d94a1dd6a4d6f9ee1bc73cbd8b9efa4f974687f1017fb736e7
+size 24944665

uploads/Maruti_Suzuki.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e298354886131e8e253df60ac25a725e140f89452ad6a2f4c0e857074107548c
+size 23558415

uploads/NTPC.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb943a9cb4db67822a4f1657454cd476e08d73259a18e990a6a36e29c78ec8aa
+size 23736915

uploads/Nestle.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2bd86b483d58de5cb6fac36113ff1851d39d14d666c33f24c8cac382f180dfa
+size 8982580

uploads/Powergrid.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb943a9cb4db67822a4f1657454cd476e08d73259a18e990a6a36e29c78ec8aa
+size 21240959

uploads/Reliance.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d7d6038b4d6957981fb422bc501ea893765c24611ea1fa8a5d3cd9425541eae
+size 12354677

uploads/SBI_Bank.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4aacda3744fdcc9aea893c0c0cd35c22e5f30a93513e37f86da29f30770961dc
+size 16162498

uploads/Sun_Pharma.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbb9d8021b365cc095ef9e5c6917e4ec4fe7cca079672ece3beb6284128c9343
+size 10123693

uploads/TCS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:992b3cb2a47a9ce84b127d23d7dc1fb22967d7592d1e8af2c2120b6764da8651
+size 10182737

uploads/Tata_Motors.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c7610ebbea4ff3dabcb1c63d673b8b2e3c6c0f45d17daf1a88908fbf8f04a25
+size 20573259

uploads/Tata_Steel.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:870be0ca867c1c562337a5319d09cac353a85dbb7907dce7c13ee47ad7af3e21
+size 26006738

uploads/Tech_Mahindra.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:304878c8f4fba07d1eb1d120556c067b46d56d23993c56b5a3f552b4f109b477
+size 17536967

uploads/Titan.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1979b0e5973d1c07ebf1a76d5580554c45acd67242746c6673a0ab276d1de6e3
+size 21978349

uploads/Ultratech.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e250887b207b20483cf040589ab9723a935dca317ce9bccd050cfdef059c0a8b
+size 29394403