Upload 5 files

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,140 @@
+import os
+from fastapi import FastAPI, Request, HTTPException, status, BackgroundTasks
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import List
+from google import genai
+from semantic_aware import load_document
+import hashlib
+import httpx
+from datetime import datetime
+import re
+from pinecone import Pinecone
+from pinecone_embeddings import PineconeVectorStore
+
+
+# Configuration
+EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
+PINECONE_INDEX = 'policy-documents'
+CACHE_DIR = "./document_cache"
+os.makedirs(CACHE_DIR, exist_ok=True)
+pinecone = Pinecone(
+    api_key=os.getenv("PINECONE_API_KEY"),
+    environment=os.getenv("PINECONE_ENV")
+)
+
+# Initialize Gemini
+# genai.configure(api_key=os.environ["GEMINI_API_KEY"])
+# model = genai.GenerativeModel('gemini-1.5-flash')
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+class QueryRequest(BaseModel):
+    documents: str
+    questions: List[str]
+
+class QueryResponse(BaseModel):
+    answers: List[str]
+
+def document_cache_key(url: str) -> str:
+    return hashlib.md5(url.encode()).hexdigest()
+
+async def fetch_with_cache(url: str) -> str:
+    """Download with caching"""
+    cache_key = document_cache_key(url)
+    cache_path = os.path.join(CACHE_DIR, f"{cache_key}.pdf")
+    
+    if os.path.exists(cache_path):
+        return cache_path
+    
+    async with httpx.AsyncClient() as client:
+        response = await client.get(url)
+        response.raise_for_status()
+        with open(cache_path, "wb") as f:
+            f.write(response.content)
+    
+    return cache_path
+
+def build_gemini_prompt(question: str, clauses: List[dict]) -> str:
+    """Strictly formatted prompt for Gemini"""
+    context = "\n\n".join(
+        f"CLAUSE {c.get('header', '')} (Page {c.get('page', 'N/A')}):\n{c['text']}"
+        for c in clauses
+    )
+    
+    return f"""You are a strict, accurate assistant that answers insurance or policy-related questions using only provided clauses.
+
+        A user has asked the following question:
+        "{question}"
+
+        You must answer only based on the given text below, without guessing or skipping any information.
+        If an answer is partially stated or implied, respond accordingly with brief clarification.
+        If the information is not present at all, reply exactly: "Not mentioned in the provided clauses."
+
+        Clauses:
+        {context}
+
+        Respond with 1 to 3 sentences max.
+        Do not add explanations, formatting, bullet points, summaries, or any output other than the answer sentence.
+
+"""
+
+# def extract_first_sentence(text: str) -> str:
+#     """Ensure single-sentence output"""
+#     sentences = re.split(r'(?<=[.!?])\s+', text.strip())
+#     return sentences[0] if sentences else text
+
+@app.post("/query", response_model=QueryResponse)
+async def answer_questions(request: Request, body: QueryRequest):
+    # Authentication
+    if request.headers.get("Authorization") != f"Bearer {os.environ['API_KEY']}":
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED)
+    
+    try:
+        # 1. Process document
+        local_path = await fetch_with_cache(body.documents)
+        doc = load_document(local_path)
+        
+        # 2. Initialize engine
+        vector_store = PineconeVectorStore(index_name=PINECONE_INDEX, pinecone=pinecone)
+        vector_store.overwrite_vectors(doc["chunks"], 'doc_a.pdf', pinecone)
+        
+        # 3. Process questions
+        answers = []
+        client = genai.Client(
+            api_key=os.environ["GEMINI_API_KEY"]
+        )
+
+        for question in body.questions:
+            # Retrieve relevant clauses
+            clauses = vector_store.retrieve_chunks(question, pinecone, top_k=5)
+        
+            # print("\n\n")
+            # print(clauses)
+
+            # Generate answer with Gemini
+            prompt = build_gemini_prompt(question, clauses)
+            response = client.models.generate_content(
+                model="gemini-2.5-flash",
+                contents=prompt
+            )
+            
+            # Strict formatting
+            # answer = extract_first_sentence(response.text)
+            # print(response.text)
+            answers.append(response.text)
+        
+        return {"answers": answers}
+        
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e)
+        )

pinecone_embeddings.py

@@ -0,0 +1,153 @@
+from pinecone import Pinecone, ServerlessSpec
+from pinecone_text.sparse import BM25Encoder # For BM25 sparse vectors
+import numpy as np
+import re
+import hashlib
+from typing import List, Dict
+import os
+import time
+
+from sentence_transformers import SentenceTransformer
+
+# Initialize Pinecone at the module level
+# It's better to initialize it once at the start of your application
+
+
+def get_index(pinecone: Pinecone, index_name: str):
+    # 1. Delete existing index if it exists
+    if index_name in pinecone.list_indexes().names():
+        print(f"Deleting existing index: {index_name}")
+        pinecone.delete_index(index_name)
+        print(f"Index {index_name} deleted.")
+    else:
+        print(f"Index {index_name} does not exist, no deletion necessary.")
+
+    # 2. Create fresh index using create_index_for_model for integrated embedding
+    print(f"Creating new index: {index_name} with integrated 'llama-text-embed-v2'")
+    pinecone.create_index( # Corrected from create_index_for_model
+        name=index_name,
+        metric="cosine", # llama-text-embed-v2 uses cosine or dotproduct
+        dimension=768, # default dimension for llama-text-embed-v2
+        # embed parameter should be at the top-level of create_index for integrated models
+        # and 'field_map' is not used directly in create_index for embedded models
+        # Instead, it's inferred from the text being passed in the 'upsert' method
+        # We will specify the embedding model when upserting.
+        spec=ServerlessSpec(cloud="aws", region="us-east-1")
+    )
+    print(f"Index {index_name} created.")
+
+    # 3. Wait for the index to be ready
+    while not pinecone.describe_index(index_name).status['ready']:
+        print("Waiting for index to be ready...")
+        time.sleep(5)
+    print(f"Index {index_name} is now ready.")
+
+    return pinecone.Index(index_name)
+
+# class EmbeddingEngine:
+#     def __init__(self, model_name: str = 'BAAI/bge-base-en-v1.5'):
+#         # For now, we are relying on Pinecone's integrated embedding for dense vectors.
+#         # This class might be used for other purposes or for local embedding generation if needed later.
+#         self.model = SentenceTransformer(model_name)
+#         self.model.max_seq_length = 512
+
+#     def encode(self, texts: List[str], batch_size: int = 32) -> np.ndarray:
+#         prefixed_texts = [
+#             f"query: {text}" if "query:" not in text.lower() else text
+#             for text in texts
+#         ]
+#         return self.model.encode(prefixed_texts, batch_size=batch_size, convert_to_numpy=True)
+
+class PineconeVectorStore:
+    def __init__(self, index_name: str, pinecone: Pinecone, dimension: int = 1024): # default dimension for llama-text-embed-v2
+        self.index_name = index_name
+        self.dimension = dimension
+        
+        self.index = get_index(pinecone, index_name)
+        
+        # Initialize BM25 encoder for sparse vectors
+        self.bm25_encoder = BM25Encoder()
+        
+        # Fit BM25 encoder on a representative corpus of your data.
+        # This is crucial for BM25's effectiveness.
+        # For this example, we'll fit on a small sample. In a real scenario,
+        # you'd fit it on a larger corpus of your document chunks.
+        print("Fitting BM25Encoder...")
+        sample_corpus = ["This is a document about machine learning.", "Another document discussing natural language processing.", "A third document focused on artificial intelligence applications."]
+        self.bm25_encoder.fit(sample_corpus) 
+        print("BM25Encoder fitted.")
+
+    def overwrite_vectors(self, document_chunks, pdf_filename: str, pinecone: Pinecone):
+        """
+        Completely replaces all vectors in the index with new data from a PDF.
+        Leverages Pinecone's integrated embedding for dense vectors and BM25 for sparse.
+        """
+        # Ensure the index is recreated before processing each new PDF
+        # self.index = get_index(pinecone, self.index_name) 
+
+        inputs = [f"query: {text['text']}" for text in document_chunks]
+
+        # embeddings = pinecone.inference.embed(
+        #     model = 'llama-text-embed-v2',
+        #     inputs = inputs,
+        #     parameters={
+        #         "input_type": "passage",
+        #         "truncate": "END"
+        #     }
+        # )
+
+        model = SentenceTransformer('BAAI/bge-base-en-v1.5')
+        embeddings = model.encode(inputs, batch_size=32, convert_to_numpy=True).tolist()
+
+        records_to_upsert = []
+        for i, chunk_text in enumerate(document_chunks):
+            # Ensure chunk_text is always a string before encoding
+
+            doc_id = hashlib.md5(f"{pdf_filename}-{chunk_text['text']}".encode('utf-8')).hexdigest()
+            sparse_vector = self.bm25_encoder.encode_documents([chunk_text["text"]])
+
+            records_to_upsert.append({
+                "id": doc_id,
+                "values": embeddings[i],
+                # "sparse_values": sparse_vector,
+                "metadata": {"text": chunk_text['text'], "header": chunk_text['header'], "page": chunk_text['page'], "type": chunk_text['type']}
+            })
+
+        batch_size = 100
+        for i in range(0, len(records_to_upsert), batch_size):
+            batch = records_to_upsert[i:i + batch_size]
+            self.index.upsert(
+                vectors=batch,
+                batch_size=batch_size
+            )
+        print(f"Successfully uploaded {len(records_to_upsert)} chunks from {pdf_filename} to Pinecone.")
+
+    def retrieve_chunks(self, query_text: str, pinecone: Pinecone, top_k: int = 5):
+        """
+        Retrieves top-k chunks based on the query using hybrid search.
+        """
+        # Generate sparse vector for the query using BM25Encoder
+        sparse_query_vector = self.bm25_encoder.encode_queries([query_text])
+
+        model = SentenceTransformer('BAAI/bge-base-en-v1.5')
+        embeddings = model.encode(f"query: {query_text}", batch_size=32, convert_to_numpy=True).tolist()
+
+        query_results = self.index.query(
+            vector=embeddings,
+            # sparse_vector=sparse_query_vector, # Include the sparse vector for hybrid search
+            top_k=top_k,
+            include_metadata=True,
+            include_values=False # No need to return the vectors themselves for RAG
+        )
+
+        retrieved_chunks = []
+        for match in query_results['matches']:
+            retrieved_chunks.append({
+                "id": match['id'],
+                "score": match['score'],
+                "text": match['metadata']['text'],
+                "header": match['metadata']['header'], 
+                "page": match['metadata']['page'], 
+                "type": match['metadata']['type']
+            })
+        return retrieved_chunks

requirements.txt

@@ -0,0 +1,19 @@
+fastapi
+uvicorn
+httpx
+requests
+pydantic
+python-multipart
+pdfplumber
+PyMuPDF
+sentence-transformers
+faiss-cpu
+huggingface-hub
+python-dotenv
+google-genai
+rank-bm25
+regex
+python-docx
+langchain
+pinecone
+pinecone-text

semantic_aware.py

@@ -0,0 +1,202 @@
+import os
+import re
+import requests
+import fitz  # PyMuPDF
+from docx import Document
+from typing import List, Dict, Optional
+from urllib.parse import urlparse
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+class DocumentLoader:
+    def __init__(self, source: str):
+        self.source = source
+        self._validate_source()
+
+    def _validate_source(self):
+        if self.source.startswith("http"):
+            try:
+                head = requests.head(self.source, timeout=5, allow_redirects=True)
+                head.raise_for_status()
+            except Exception as e:
+                raise ValueError(f"URL inaccessible: {str(e)}")
+        elif not os.path.exists(self.source):
+            raise FileNotFoundError(f"Local file not found: {self.source}")
+
+    def extract(self) -> List[Dict]:
+        raise NotImplementedError("Subclasses must implement extract()")
+
+class PDFLoader(DocumentLoader):
+    def __init__(self, source: str):
+        super().__init__(source)
+        self.is_url = self.source.startswith("http")
+        self.header_pattern = re.compile(
+            r"^(§|\d+\.\d+|\bARTICLE\b|\bCLAUSE\b|\bSECTION\b)", 
+            re.IGNORECASE
+        )
+
+    def _download_pdf(self) -> str:
+        local_path = "temp_blob.pdf"
+        try:
+            response = requests.get(self.source, timeout=10, stream=True)
+            response.raise_for_status()
+            with open(local_path, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            return local_path
+        except Exception as e:
+            if os.path.exists(local_path):
+                os.remove(local_path)
+            raise RuntimeError(f"Download failed: {str(e)}")
+
+    def _extract_with_structure(self, pdf_path: str) -> str:
+        full_text = []
+        try:
+            with fitz.open(pdf_path) as doc:
+                for page_num, page in enumerate(doc, start=1):
+                    text = page.get_text("text", flags=fitz.TEXT_PRESERVE_LIGATURES)
+                    page_header = f"\n[PAGE {page_num}]\n"
+                    full_text.append(page_header + text)
+        except Exception as e:
+            raise RuntimeError(f"PDF parsing failed: {str(e)}")
+        return "\n".join(full_text)
+
+    def _chunk_text(self, text: str) -> List[Dict]:
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000,
+            chunk_overlap=200,
+            separators=[
+                "\n§", "\nArticle", "\nClause", "\nSECTION", 
+                "\nSubsection", "\n\n", "\n", " ", ""
+            ]
+        )
+        
+        chunks = splitter.split_text(text)
+        structured_chunks = []
+        
+        for chunk in chunks:
+            header_match = self.header_pattern.search(chunk)
+            header = header_match.group(0).strip() if header_match else "General"
+            page_match = re.search(r"\[PAGE (\d+)\]", chunk)
+            page = int(page_match.group(1)) if page_match else 1
+            
+            structured_chunks.append({
+                "text": chunk,
+                "header": header,
+                "page": page,
+                "type": "clause" if header_match else "text_block"
+            })
+            
+        return structured_chunks
+
+    def _extract_tables(self, pdf_path: str) -> List[Dict]:
+        tables = []
+        try:
+            with fitz.open(pdf_path) as doc:
+                for page_num, page in enumerate(doc, start=1):
+                    for table in page.find_tables():
+                        table_data = []
+                        for row in table.extract():
+                            # Filter None values and convert to strings
+                            cleaned_row = [str(cell) if cell is not None else "" for cell in row]
+                            table_data.append("|".join(cleaned_row))
+                        
+                        if table_data:  # Only add non-empty tables
+                            tables.append({
+                                "text": f"[TABLE {page_num}.{len(tables)+1}]\n" + "\n".join(table_data),
+                                "header": f"Table {page_num}.{len(tables)+1}",
+                                "page": page_num,
+                                "type": "table"
+                            })
+        except Exception as e:
+            print(f"Table extraction warning: {str(e)}")
+        return tables
+
+    def extract(self) -> List[Dict]:
+        pdf_path = self._download_pdf() if self.is_url else self.source
+        try:
+            full_text = self._extract_with_structure(pdf_path)
+            chunks = self._chunk_text(full_text)
+            tables = self._extract_tables(pdf_path)
+            return chunks + tables
+        finally:
+            if self.is_url and os.path.exists(pdf_path):
+                try:
+                    os.remove(pdf_path)
+                except PermissionError:
+                    # Handle file lock issues on Windows
+                    import time
+                    time.sleep(0.1)
+                    os.remove(pdf_path)
+
+class DOCXLoader(DocumentLoader):
+    def extract(self) -> List[Dict]:
+        chunks = []
+        current_header = None
+        
+        try:
+            doc = Document(self.source)
+            for para in doc.paragraphs:
+                text = para.text.strip()
+                if not text:
+                    continue
+                    
+                if para.style.name.lower().startswith(('heading', 'title')):
+                    current_header = text
+                    continue
+                    
+                chunks.append({
+                    "text": text,
+                    "header": current_header or "General",
+                    "style": para.style.name,
+                    "type": "heading" if current_header else "paragraph"
+                })
+        except Exception as e:
+            raise RuntimeError(f"DOCX processing failed: {str(e)}")
+            
+        return chunks
+
+def load_document(source: str) -> Dict:
+    def _is_pdf(content: bytes) -> bool:
+        return content[:4] == b'%PDF'
+
+    def _is_docx(content: bytes) -> bool:
+        return (b'word/_rels' in content or 
+                b'[Content_Types].xml' in content)
+
+    try:
+        # Content-based detection
+        if source.startswith("http"):
+            response = requests.get(source, stream=True)
+            response.raise_for_status()
+            sample = response.raw.read(1024)
+        else:
+            with open(source, 'rb') as f:
+                sample = f.read(1024)
+
+        if _is_pdf(sample):
+            loader = PDFLoader(source)
+        elif _is_docx(sample):
+            loader = DOCXLoader(source)
+        else:
+            raise ValueError("Unrecognized file format")
+
+    except Exception as e:
+        # Extension fallback
+        ext = os.path.splitext(urlparse(source).path if source.startswith("http") else source)[1].lower()
+        if ext == '.pdf':
+            loader = PDFLoader(source)
+        elif ext == '.docx':
+            loader = DOCXLoader(source)
+        else:
+            raise ValueError(f"Unsupported file type (extension: {ext})")
+
+    return {
+        "source": source,
+        "chunks": loader.extract()
+    }
+
+if __name__ == '__main__':
+
+    output = load_document('https://hackrx.blob.core.windows.net/assets/hackrx_6/policies/CHOTGDP23004V012223.pdf?sv=2023-01-03&st=2025-07-30T06%3A46%3A49Z&se=2025-09-01T06%3A46%3A00Z&sr=c&sp=rl&sig=9szykRKdGYj0BVm1skP%2BX8N9%2FRENEn2k7MQPUp33jyQ%3D')
+    print("hello")
+    print(len(output['chunks']))