add app

.env copy

@@ -0,0 +1,5 @@
+OPENAI_API_KEY=Your_OPENAI_API_KEY
+POSTGRES_URL_EMBEDDINDS=YOUR_POSTGRES_URL
+POSTGRES_URL=YOUR_POSTGRES_URL
+PINECONE_API_KEY = YOUR_PINECONE_API_KEY
+#POSTGRES_URL_SERVER=YOUR_POSTGRES_URL_SERVER

.gitignore

@@ -1,2 +1,13 @@
 .env
-venv/
+venv/
+.venv/
+myvenv/
+my-env/
+.env
+
+data/
+data_ingest/
+__pycache__
+
+images/
+*.ipynb

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["streamlit", "run", "app.py", "--server.address=0.0.0.0", "--server.port=7860"]

README.md

@@ -0,0 +1,10 @@
+---
+title: Stramlit Pdf
+emoji: 📚
+colorFrom: blue
+colorTo: yellow
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,90 @@
+import streamlit as st
+import random
+from langchain_components.replier import get_context_from_vectorstore,get_vectorstore_from_postgres,prepare_prompt_and_chain_with_history,get_vectorstore_from_pinecone
+import fitz
+
+def display_pdf(pdf_path):
+    try:
+        pdf_document = fitz.open(pdf_path)
+        num_pages = pdf_document.page_count
+        st.sidebar.write(f"Total pages: {num_pages}")
+
+        for page_num in range(num_pages):
+            page = pdf_document.load_page(page_num)
+            image = page.get_pixmap()
+            st.sidebar.image(image.tobytes(), caption=f"Page {page_num + 1}", use_column_width=True)
+
+    except Exception as e:
+        st.sidebar.error(f"Error loading PDF: {e}")
+
+
+
+def main():
+    st.header('Interact with your PDF that includes images, tables, and graphs.')
+    
+    if "activate_chat" not in st.session_state:
+        st.session_state.activate_chat = False
+
+    if "messages" not in st.session_state:
+        st.session_state.messages = []
+
+
+    with st.sidebar:
+        username = st.text_input("Please enter your name here")
+        if st.button('Press Button to Start chat with Narendra AI'):
+            if "user_id" not in st.session_state:
+                st.session_state.user_id = username
+            
+            if "session_id" not in st.session_state:
+                random_number = random.randint(1, 1000000)
+                st.session_state.session_id = str(random_number)
+
+            if "vectorstore" not in st.session_state:
+                collection_name="fy2024_chunk_2000"
+                pinecone_collection_name="fy2024"
+                st.session_state.vectorstore = get_vectorstore_from_postgres(collection_name)
+                #st.session_state.vectorstore = get_vectorstore_from_pinecone(pinecone_collection_name)
+
+            if "chain" not in st.session_state:
+                st.session_state.chain = prepare_prompt_and_chain_with_history()
+
+            st.session_state.activate_chat = True
+
+
+        st.subheader("PDF Viewer")
+        pdf_path = "data/fy2024.pdf"
+        if st.button('Show PDF'):
+            st.session_state.pdf_path = pdf_path
+
+    if "pdf_path" in st.session_state:
+        pdf_path = st.session_state.pdf_path
+        display_pdf(pdf_path)
+
+
+
+    for message in st.session_state.messages:
+        with st.chat_message(message["role"], avatar = message['avatar']):
+            st.markdown(message["content"])
+
+    if st.session_state.activate_chat == True:
+        if prompt := st.chat_input("Ask your question from the PDF? "):
+            with st.chat_message("user", avatar = '👨🏻'):
+                st.markdown(prompt)
+            st.session_state.messages.append({"role": "user",  "avatar" :'👨🏻', "content": prompt})
+            
+            user_id = st.session_state.user_id
+            session_id = st.session_state.session_id
+            vectorstore = st.session_state.vectorstore
+            chain = st.session_state.chain
+            print("chain Done")
+
+            data=get_context_from_vectorstore(vectorstore,prompt)
+            ai_msg =chain.invoke({"data": data, "input": prompt}, config={"configurable": {"user_id": user_id, "session_id": session_id}})
+            cleaned_response=ai_msg.content
+            with st.chat_message("assistant", avatar='🤖'):
+                st.markdown(cleaned_response)
+            st.session_state.messages.append({"role": "assistant",  "avatar" :'🤖', "content": cleaned_response})
+
+
+if __name__ == '__main__':
+    main()

langchain_components/ingest.py

@@ -0,0 +1,213 @@
+import os
+import uuid
+import base64
+from unstructured.partition.pdf import partition_pdf
+from langchain_openai import ChatOpenAI
+from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
+from langchain.schema.messages import HumanMessage, SystemMessage
+from langchain.schema.document import Document
+from langchain_openai import OpenAIEmbeddings
+from langchain_postgres.vectorstores import PGVector
+from pinecone import Pinecone
+from pinecone import ServerlessSpec
+from langchain_pinecone import PineconeVectorStore
+
+from dotenv import load_dotenv
+load_dotenv()
+
+openai_api_key = os.getenv("OPENAI_API_KEY")
+POSTGRES_URL_EMBEDDINDS=os.getenv("POSTGRES_URL_EMBEDDINDS")
+PINECONE_API_KEY=os.getenv("PINECONE_API_KEY")
+
+
+
+filename="/home/bluebash-005/code/bluebash/poc/stramlit_pdf/data/fy2024.pdf"
+output_path = "/home/bluebash-005/code/bluebash/poc/stramlit_pdf/images"
+openai_ef = OpenAIEmbeddings()
+
+
+
+text_elements = []
+text_summaries = []
+
+table_elements = []
+table_summaries = []
+
+image_elements = []
+image_summaries = []
+
+
+def file_reader():
+    raw_pdf_elements = partition_pdf(
+        filename=filename,
+        extract_images_in_pdf=True,
+        infer_table_structure=True,
+        chunking_strategy="by_title",
+        max_characters=4000,
+        new_after_n_chars=3800,
+        combine_text_under_n_chars=2000,
+        extract_image_block_output_dir=output_path,
+    )
+    return raw_pdf_elements
+
+
+
+def text_insert(raw_pdf_elements):
+    summary_prompt = """
+    Summarize the following {element_type}:
+    {element}
+    """
+
+    prompt=PromptTemplate.from_template(summary_prompt)
+    llm=ChatOpenAI(model="gpt-3.5-turbo", openai_api_key = openai_api_key, max_tokens=1024)
+    runnable = prompt | llm
+
+    for e in raw_pdf_elements:
+        if 'CompositeElement' in repr(e):
+            text_elements.append(e.text)
+            summary = runnable.invoke({'element_type': 'text', 'element': e})
+            text_summaries.append(summary.content)
+
+        elif 'Table' in repr(e):
+            table_elements.append(e.text)
+            summary = runnable.invoke({'element_type': 'table', 'element': e})
+            table_summaries.append(summary.content)
+
+
+def image_insert():
+
+    def encode_image(image_path):
+        with open(image_path, "rb") as f:
+            return base64.b64encode(f.read()).decode('utf-8')
+
+    def summarize_image(encoded_image):
+        prompt = [
+            SystemMessage(content="You are a bot that is good at analyzing images."),
+            HumanMessage(content=[
+                {
+                    "type": "text",
+                    "text": "Describe the contents of this image."
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/jpeg;base64,{encoded_image}"
+                    },
+                },
+            ])
+        ]
+        response = ChatOpenAI(model="gpt-4-vision-preview", openai_api_key=openai_api_key, max_tokens=1024).invoke(prompt)
+        return response.content
+    
+
+    for i in os.listdir(output_path):
+        if i.endswith(('.png', '.jpg', '.jpeg')):
+            image_path = os.path.join(output_path, i)
+            encoded_image = encode_image(image_path)
+            image_elements.append(encoded_image)
+            summary = summarize_image(encoded_image)
+            image_summaries.append(summary)
+
+
+documents = []
+retrieve_contents = []
+
+def get_docummets():
+    for e, s in zip(text_elements, text_summaries):
+        i = str(uuid.uuid4())
+        doc = Document(
+            page_content = s,
+            metadata = {
+                'id': i,
+                'type': 'text',
+                'original_content': e
+            }
+        )
+        retrieve_contents.append((i, e))
+        documents.append(doc)
+    print("text_element done")
+
+    for e, s in zip(table_elements, table_summaries):
+        doc = Document(
+            page_content = s,
+            metadata = {
+                'id': i,
+                'type': 'table',
+                'original_content': e
+            }
+        )
+        retrieve_contents.append((i, e))
+        documents.append(doc)
+    
+    print("table_elements done")
+
+    for e, s in zip(image_elements, image_summaries):
+        doc = Document(
+            page_content = s,
+            metadata = {
+                'id': i,
+                'type': 'image',
+                'original_content': e
+            }
+        )
+        retrieve_contents.append((i, s))
+        documents.append(doc)
+
+    print("image_elements Done")
+
+def add_docs_to_postgres(collection_name):
+    vectorstore = PGVector(embeddings=openai_ef,collection_name=collection_name,connection=POSTGRES_URL_EMBEDDINDS,use_jsonb=True,)
+    vectorstore.add_documents(documents)
+
+
+
+def add_docs_to_pinecone(index_name):
+    pc = Pinecone(api_key=PINECONE_API_KEY)
+
+    spec = ServerlessSpec(cloud='aws', region='us-east-1')
+    if index_name in pc.list_indexes().names():
+        pc.delete_index(index_name)
+
+    # we create a new index
+    pc.create_index(
+            index_name,
+            dimension=1536,
+            metric='dotproduct',
+            spec=spec
+        )
+    import pdb
+    pdb.set_trace()
+    n=len(documents)//2
+    doc1=documents[:n]
+    doc2=documents[n:]
+
+    vectorstore_from_docs = PineconeVectorStore.from_documents(
+        doc1,
+        index_name=index_name,
+        embedding=openai_ef
+    )
+
+
+
+
+def main():
+    collection_name="fy2024"
+    print("started file reader")
+    raw_pdf_elements=file_reader()
+    print(raw_pdf_elements)
+    print()
+
+    text_insert(raw_pdf_elements)
+    print("text_insert Done")
+    image_insert()
+    print("image_insert Done")
+    print()
+    get_docummets()
+    print("get_docummets Done")
+    #add_docs_to_postgres(collection_name)
+    add_docs_to_pinecone(collection_name)
+    print("Done")
+
+if __name__=="__main__":
+    main()

langchain_components/replier.py

@@ -0,0 +1,103 @@
+import os
+from langchain_openai import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.runnables.history import RunnableWithMessageHistory
+from langchain_core.runnables import  ConfigurableFieldSpec
+from langchain_community.chat_message_histories import PostgresChatMessageHistory
+from langchain_openai import OpenAIEmbeddings
+from langchain_postgres.vectorstores import PGVector
+from langchain_community.vectorstores import Pinecone
+
+import logging
+from dotenv import load_dotenv
+load_dotenv()
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+os.environ["OPENAI_API_KEY"] =os.getenv("OPENAI_API_KEY")
+POSTGRES_URL = os.getenv("POSTGRES_URL")
+
+def create_postgres_chat_message_history(session_id, user_id):
+    return PostgresChatMessageHistory(connection_string=POSTGRES_URL,session_id=session_id)
+
+def prepare_prompt_and_chain_with_history():
+    llm = ChatOpenAI(model="gpt-4o")
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                """You are an expert in data that hepls to summerize the graph and extract information from that images. Answer the question based only on the following context, which can include text, images and tables: "
+                ** IMPORTANT INSTRUCTIONS -->
+                    1. Reply in 50 words maximum.
+                    2. Only answer the question related to the context else say you don't know the answer.""",
+
+            ),
+            "Here is the context to answer user's questions everytime --> {data}. ",
+            MessagesPlaceholder(variable_name="history"),
+            ("user", "{input}"),
+        ]
+    )
+    runnable = prompt | llm
+    with_message_history = RunnableWithMessageHistory(
+            runnable,
+            create_postgres_chat_message_history,
+            input_messages_key="input",
+            history_messages_key="history",
+            history_factory_config=[
+                ConfigurableFieldSpec(
+                    id="user_id",
+                    annotation=str,
+                    name="User ID",
+                    description="Unique identifier for the user.",
+                    default="",
+                    is_shared=True,
+                ),
+                ConfigurableFieldSpec(
+                    id="session_id",
+                    annotation=str,
+                    name="Session ID",
+                    description="Unique identifier for the conversation.",
+                    default="",
+                    is_shared=True,
+                ),
+            ],
+            verbose=True,
+        )
+    return with_message_history
+
+
+
+
+def get_vectorstore_from_postgres(collection_name):
+    openai_ef = OpenAIEmbeddings()
+    vectorstore = PGVector(
+        embeddings=openai_ef,
+        collection_name=collection_name,
+        connection=POSTGRES_URL,
+        use_jsonb=True,
+    ) 
+    return vectorstore      
+
+
+def get_vectorstore_from_pinecone(index_name):
+    openai_ef = OpenAIEmbeddings()
+    vectorstore = Pinecone.from_existing_index(index_name, openai_ef)
+    return vectorstore
+
+
+def get_context_from_vectorstore(vectorstore,user_query):
+    logging.info("Start postgres vector search......")
+    relevant_docs = vectorstore.similarity_search(user_query,k=4)
+    logging.info(relevant_docs)
+    context = ""
+    relevant_images = []
+    for d in relevant_docs:
+        if d.metadata['type'] == 'text':
+            context += '[text]' + d.metadata['original_content']
+        elif d.metadata['type'] == 'table':
+            context += '[table]' + d.metadata['original_content']
+        elif d.metadata['type'] == 'image':
+            context += '[image]' + d.page_content
+            relevant_images.append(d.metadata['original_content'])
+    return context

pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "streamlit-unstructed-pdf"
+version = "0.1.0"
+description = ""
+authors = ["narendra-bluebash <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.9.7 || >3.9.7,<4.0"
+streamlit = "1.35.0"
+langchain-openai = "^0.1.9"
+langchain = "^0.2.5"
+langchain-community = "^0.2.5"
+langchain-postgres = "^0.0.9"
+python-dotenv = "^1.0.1"
+psycopg2-binary = "^2.9.9"
+pinecone-client = "^4.1.1"
+pymupdf = "^1.24.6"
+
+[tool.poetry.group.dev.dependencies]
+ipykernel = "^6.29.4"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,112 @@
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.4.0
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.2.0
+blinker==1.8.2
+cachetools==5.3.3
+certifi==2024.6.2
+charset-normalizer==3.3.2
+click==8.1.7
+comm==0.2.2
+dataclasses-json==0.6.7
+debugpy==1.8.1
+decorator==5.1.1
+distro==1.9.0
+exceptiongroup==1.2.1
+executing==2.0.1
+frozenlist==1.4.1
+gitdb==4.0.11
+GitPython==3.1.43
+greenlet==3.0.3
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+idna==3.7
+importlib_metadata==7.2.1
+ipykernel==6.29.4
+ipython==8.18.1
+jedi==0.19.1
+Jinja2==3.1.4
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+langchain==0.2.5
+langchain-community==0.2.5
+langchain-core==0.2.9
+langchain-openai==0.1.9
+langchain-postgres==0.0.9
+langchain-text-splitters==0.2.1
+langsmith==0.1.82
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+marshmallow==3.21.3
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+multidict==6.0.5
+mypy-extensions==1.0.0
+nest-asyncio==1.6.0
+numpy==1.26.4
+openai==1.35.3
+orjson==3.10.5
+packaging==24.1
+pandas==2.2.2
+parso==0.8.4
+pexpect==4.9.0
+pgvector==0.2.5
+pillow==10.3.0
+pinecone-client==4.1.1
+pinecone-plugin-interface==0.0.7
+platformdirs==4.2.2
+prompt_toolkit==3.0.47
+protobuf==4.25.3
+psutil==6.0.0
+psycopg==3.1.19
+psycopg-pool==3.2.2
+psycopg2-binary==2.9.9
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==16.1.0
+pydantic==2.7.4
+pydantic_core==2.18.4
+pydeck==0.9.1
+Pygments==2.18.0
+PyMuPDF==1.24.6
+PyMuPDFb==1.24.6
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.1
+PyYAML==6.0.1
+pyzmq==26.0.3
+referencing==0.35.1
+regex==2024.5.15
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.18.1
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.1
+SQLAlchemy==2.0.31
+stack-data==0.6.3
+streamlit==1.35.0
+tenacity==8.4.2
+tiktoken==0.7.0
+toml==0.10.2
+toolz==0.12.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+typing-inspect==0.9.0
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.2
+watchdog==4.0.1
+wcwidth==0.2.13
+yarl==1.9.4
+zipp==3.19.2