Upload 9 files

README.md

@@ -1,13 +1,7 @@
----
-title: LawGPT
-emoji: 🔥
-colorFrom: purple
-colorTo: red
-sdk: gradio
-sdk_version: 3.38.0
-app_file: app.py
-pinned: false
-license: mit
----
+# Question Answer over a PDF document
+This is a repository for my [YouTube channel](https://www.youtube.com/channel/UCOT01XvBSj12xQsANtTeAcQ), you can build a 
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+#### YouTube video
+I have created a video explaining this repository, you can watch it [here](https://youtu.be/fNfcEvhwTMA).
+
+It have Gradio UI as well, [here](https://youtu.be/XRfHDoROopY) is a video explaining it.

config/config.py

@@ -0,0 +1,18 @@
+import os
+import tempfile
+
+from dotenv import load_dotenv, find_dotenv
+
+load_dotenv(find_dotenv())
+
+PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
+PINECONE_ENVIRONMENT = os.getenv('PINECONE_ENVIRONMENT')
+OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
+
+OUTPUT_DIR = os.path.join(
+    tempfile.gettempdir(),
+    'rkk-document-gpt',
+    'output'
+)
+
+os.makedirs(OUTPUT_DIR, exist_ok=True)

conversation.py

@@ -0,0 +1,38 @@
+from langchain.chains import ConversationalRetrievalChain
+from langchain.chat_models import ChatOpenAI
+from langchain.vectorstores import Pinecone
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.memory import ConversationBufferMemory
+import pinecone
+
+
+def create_conversation(query: str, chat_history: list, pinecone_api_key: str, pinecone_environment: str, pinecone_index_name: str, openai_api_key: str) -> tuple:
+    try:
+        pinecone.init(
+            api_key=pinecone_api_key,
+            environment=pinecone_environment,
+        )
+        embeddings = OpenAIEmbeddings(
+            openai_api_key=openai_api_key
+        )
+        db = Pinecone.from_existing_index(
+            index_name=pinecone_index_name,
+            embedding=embeddings
+        )
+        memory = ConversationBufferMemory(
+            memory_key='chat_history',
+            return_messages=False
+        )
+        cqa = ConversationalRetrievalChain.from_llm(
+            llm=ChatOpenAI(temperature=0.0,
+                           openai_api_key=openai_api_key),
+            retriever=db.as_retriever(),
+            memory=memory,
+            get_chat_history=lambda h: h,
+        )
+        result = cqa({'question': query, 'chat_history': chat_history})
+        chat_history.append((query, result['answer']))
+        return '', chat_history
+    except Exception as e:
+        chat_history.append((query, e))
+        return '', chat_history

gradio-ui.py

@@ -0,0 +1,41 @@
+import gradio as gr
+
+from indexes import create_indexes, clear_indexes
+from conversation import create_conversation
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        pinecone_api_key = gr.components.Textbox(
+            label='Pinecone API key', type='password')
+        pinecone_environment = gr.components.Textbox(
+            label='Pinecone environment')
+        pinecone_index_name = gr.components.Textbox(
+            label='Pinecone index name')
+        openai_api_key = gr.components.Textbox(
+            label='Openai API key', type='password')
+    with gr.Row():
+        with gr.Column():
+            file = gr.components.File(
+                label='Upload your pdf file',
+                file_count='single',
+                file_types=['.pdf'])
+            with gr.Row():
+                upload = gr.components.Button(
+                    value='Upload', variant='primary')
+                index_clear_btn = gr.components.Button(
+                    value='Clear', variant='stop')
+        label = gr.components.Textbox(label='Status of uploaded file')
+
+    chatbot = gr.Chatbot(label='Talk to the Document')
+    msg = gr.Textbox(label='Ask a question')
+    clear = gr.ClearButton([msg, chatbot])
+
+    upload.click(create_indexes, [
+                 file, pinecone_api_key, pinecone_environment, pinecone_index_name, openai_api_key], [label])
+    index_clear_btn.click(clear_indexes, [
+                pinecone_api_key, pinecone_environment, pinecone_index_name], [label, file])
+    msg.submit(create_conversation, [msg, chatbot, pinecone_api_key,
+               pinecone_environment, pinecone_index_name, openai_api_key], [msg, chatbot])
+
+if __name__ == '__main__':
+    demo.launch()

indexes.py

@@ -0,0 +1,52 @@
+import tempfile
+import os
+from langchain.document_loaders import PyPDFLoader
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.document_loaders import DirectoryLoader, TextLoader
+from langchain.vectorstores import Pinecone
+from PyPDF2 import PdfReader
+import pinecone
+
+from config import config
+
+def create_indexes(file: tempfile, pinecone_api_key: str, pinecone_environment: str, pinecone_index_name: str, openai_api_key: str) -> str:
+    try:
+        file_path = file.name
+        reader = PyPDFLoader(file_path)
+        documents = reader.load_and_split()
+        embeddings = OpenAIEmbeddings(
+            openai_api_key=openai_api_key
+        )
+        pinecone.init(
+            api_key=pinecone_api_key,
+            environment=pinecone_environment
+        )
+        indexes_list = pinecone.list_indexes()
+        if pinecone_index_name not in indexes_list:
+            pinecone.create_index(
+                name=pinecone_index_name,
+                dimension=1536
+            )
+        Pinecone.from_documents(
+            documents=documents,
+            embedding=embeddings,
+            index_name=pinecone_index_name
+        )
+        return 'Document uploaded and index created successfully. You can chat now.'
+    except Exception as e:
+        return e
+
+def clear_indexes(pinecone_api_key: str, pinecone_environment: str, pinecone_index_name: str) -> str:
+    try:
+        pinecone.init(
+            api_key=pinecone_api_key,
+            environment=pinecone_environment
+        )
+        indexes_list = pinecone.list_indexes()
+        if pinecone_index_name in indexes_list:
+            pinecone.delete_index(name=pinecone_index_name)
+        return 'Indexes cleared.', None
+    except Exception as e:
+        return e, None
+    

requirements.txt

@@ -0,0 +1,142 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+anyio==3.7.1
+asttokens==2.2.1
+async-timeout==4.0.2
+attrs==23.1.0
+autopep8==2.0.2
+backcall==0.2.0
+backoff==2.2.1
+certifi==2023.5.7
+charset-normalizer==3.2.0
+click==8.1.4
+clickhouse-connect==0.6.6
+coloredlogs==15.0.1
+comm==0.1.3
+contourpy==1.1.0
+cycler==0.11.0
+dataclasses-json==0.5.9
+debugpy==1.6.7
+decorator==5.1.1
+dnspython==2.3.0
+duckdb==0.8.1
+exceptiongroup==1.1.2
+executing==1.2.0
+fastapi==0.100.0
+ffmpy==0.3.0
+filelock==3.12.2
+flatbuffers==23.5.26
+fonttools==4.40.0
+frozenlist==1.3.3
+fsspec==2023.6.0
+gradio==3.36.1
+gradio_client==0.2.7
+greenlet==2.0.2
+h11==0.14.0
+hnswlib==0.7.0
+httpcore==0.17.3
+httptools==0.6.0
+httpx==0.24.1
+huggingface-hub==0.16.4
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.0.0
+ipykernel==6.24.0
+ipython==8.14.0
+ipywidgets==8.0.7
+jedi==0.18.2
+Jinja2==3.1.2
+jsonschema==4.18.0
+jsonschema-specifications==2023.6.1
+jupyter_client==8.3.0
+jupyter_core==5.3.1
+jupyterlab-widgets==3.0.8
+kiwisolver==1.4.4
+langchain==0.0.228
+langchainplus-sdk==0.0.20
+linkify-it-py==2.0.2
+loguru==0.7.0
+lz4==4.3.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.3
+marshmallow==3.19.0
+marshmallow-enum==1.5.1
+matplotlib==3.7.2
+matplotlib-inline==0.1.6
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+nest-asyncio==1.5.6
+numexpr==2.8.4
+numpy==1.25.1
+onnxruntime==1.15.1
+openai==0.27.8
+openapi-schema-pydantic==1.2.4
+orjson==3.9.2
+overrides==7.3.1
+packaging==23.1
+pandas==2.0.3
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==10.0.0
+pinecone-client==2.2.2
+platformdirs==3.8.1
+posthog==3.0.1
+prompt-toolkit==3.0.39
+protobuf==4.23.4
+psutil==5.9.5
+ptyprocess==0.7.0
+pulsar-client==3.2.0
+pure-eval==0.2.2
+pycodestyle==2.10.0
+pydantic==1.10.11
+pydub==0.25.1
+Pygments==2.15.1
+pyparsing==3.0.9
+PyPDF2==3.0.1
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+pyzmq==25.1.0
+referencing==0.29.1
+regex==2023.6.3
+requests==2.31.0
+rpds-py==0.8.10
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy==2.0.18
+stack-data==0.6.2
+starlette==0.27.0
+sympy==1.12
+tenacity==8.2.2
+tiktoken==0.4.0
+tokenizers==0.13.3
+tomli==2.0.1
+toolz==0.12.0
+tornado==6.3.2
+tqdm==4.65.0
+traitlets==5.9.0
+typing-inspect==0.9.0
+typing_extensions==4.7.1
+tzdata==2023.3
+uc-micro-py==1.0.2
+urllib3==2.0.3
+uvicorn==0.22.0
+watchfiles==0.19.0
+wcwidth==0.2.6
+websockets==11.0.3
+widgetsnbextension==4.0.8
+yarl==1.9.2
+zipp==3.16.0
+zstandard==0.21.0
+pypdf==3.12.2