Create app.py

app.py

@@ -0,0 +1,168 @@
+import os
+from dotenv import load_dotenv
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.schema import HumanMessage
+from langchain_openai import OpenAIEmbeddings
+from langchain_voyageai import VoyageAIEmbeddings
+from langchain_pinecone import PineconeVectorStore
+from langchain_openai import ChatOpenAI
+from langchain.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from typing import List, Tuple
+from langchain.schema import BaseRetriever
+from langchain_core.documents import Document
+from langchain_core.runnables import chain
+import gradio as gr
+from pinecone import Pinecone, ServerlessSpec 
+import openai
+    
+# Load environment variables
+load_dotenv()
+
+# Initialize OpenAI and Pinecone credentials
+openai.api_key = os.environ.get("OPENAI_API_KEY")
+pinecone_api_key = os.environ.get("PINECONE_API_KEY")
+pinecone_environment = os.environ.get("PINECONE_ENV")
+voyage_api_key = os.environ.get("VOYAGE_API_KEY")
+pinecone_index_name = "rag-proto011"
+
+# Initialize Pinecone
+pc = Pinecone(api_key=pinecone_api_key)
+if pinecone_index_name not in pc.list_indexes().names():
+    pc.create_index(
+        name=pinecone_index_name, 
+        dimension=1024, #1024- voyage-law-2,  # '1536' is the dimension for ada-002 embeddings
+        metric='cosine',
+        spec=ServerlessSpec(
+            cloud='aws',
+            region=pinecone_environment
+        )
+    )
+    print("Pinecone Index provisioned")
+else:
+    print("Pinecone Index already provisioned")
+
+# Initialize embeddings
+#embeddings = OpenAIEmbeddings(model="text-embedding-ada-002")
+embeddings = VoyageAIEmbeddings(
+    voyage_api_key=voyage_api_key, model="voyage-law-2"
+)
+
+def search_documents(query):
+    try:
+        # Initialize the vector store and retriever
+        vector_store = PineconeVectorStore(index_name=pinecone_index_name, embedding=embeddings)
+        
+        # Use maxMarginalRelevanceSearch to improve diversity in results
+        results = vector_store.max_marginal_relevance_search(query, k=7, fetch_k=20)  # Adjust fetch_k for more diverse results
+        
+        # Filter results to ensure uniqueness based on metadata.id
+        seen_ids = set()
+        unique_results = []
+        for result in results:
+            unique_id = result.metadata.get("id")
+            if unique_id and unique_id not in seen_ids:
+                seen_ids.add(unique_id)
+                unique_results.append(result)
+
+        # Collect relevant context from unique results
+        context = []
+        for result in unique_results:
+            context.append({
+                "doc_id": result.metadata.get("doc_id", "N/A"),
+                "chunk_id": result.metadata.get("id", "N/A"),
+                "title": result.metadata.get("source", "N/A"),
+                "relevant_text": result.page_content,
+                "page_number": result.metadata.get("page", "N/A"),
+                "score": result.metadata.get("score", 0.0),  # Score might not be available in all libraries
+            })
+        
+        # Combine the relevant text for additional processing, if needed
+        combined_context = "\n\n".join([res["relevant_text"] for res in context])
+        return context, combined_context
+    except Exception as e:
+        return [], f"Error searching documents: {str(e)}"
+
+
+def generate_output(context, query):
+    try:
+        llm = ChatOpenAI(model="gpt-4", openai_api_key=openai.api_key, temperature=0.7)
+        prompt_template = PromptTemplate(
+            template="""
+            Use the following context to answer the question as accurately as possible:
+            Context: {context}
+            Question: {question}
+            Answer:""",
+            input_variables=["context", "question"]
+        )
+        prompt = prompt_template.format(context=context, question=query)
+        response = llm([HumanMessage(content=prompt)])
+        return response.content
+    except Exception as e:
+        return f"Error generating output: {str(e)}"
+
+def complete_workflow(query):
+    try:
+        context_data, combined_context = search_documents(query)
+        #natural_language_output = generate_output(combined_context, query)
+        
+        document_titles = list({os.path.basename(doc["title"]) for doc in context_data})  # Get only file names
+
+        formatted_titles = " " + "\n".join(document_titles)
+
+        results = {
+            "results": [
+                {
+                    "natural_language_output": generate_output(doc["relevant_text"], query),
+                    "doc_id": doc["doc_id"],
+                    "chunk_id": doc["chunk_id"],
+                    "title": doc["title"],
+                    "relevant_text": doc["relevant_text"],
+                    "page_number": doc["page_number"],
+                    "score": doc["score"],
+                }
+                for doc in context_data
+            ]
+        }
+
+        return results, formatted_titles  # Return results and formatted document titles
+    except Exception as e:
+        return {"results": []}, f"Error in workflow: {str(e)}"
+
+
+def delete_index():
+    try:
+        pinecone_index_name = "rag-proto011"
+        if pinecone_index_name in pc.list_indexes().names():
+            pc.delete_index(name=pinecone_index_name)
+            return "Pinecone Index Deleted"
+        else:
+            return "Pinecone Index Had Already Been Deleted"
+    except Exception as e:
+        return f"Error deleting Pinecone index: {str(e)}"
+
+
+def gradio_app():
+    with gr.Blocks(css=".result-output {width: 150%; font-size: 16px; padding: 10px;}") as app:
+        gr.Markdown("### Intelligent Document Search Prototype-v0.1.2 ")
+        
+        with gr.Row():
+            user_query = gr.Textbox(label="Enter Your Search Query")
+            search_btn = gr.Button("Search")
+
+        with gr.Row():
+            result_output = gr.JSON(label="Search Results", elem_id="result-output")
+        with gr.Row():
+            titles_output = gr.Textbox(label="Document Titles", interactive=False)  # New Textbox for Titles
+        
+        search_btn.click(
+            complete_workflow,
+            inputs=user_query,
+            outputs=[result_output, titles_output],  
+        )
+
+
+    return app
+# Launch the app
+gradio_app().launch()