Upload 39 files

.gitignore

@@ -0,0 +1 @@
+.env

Dockerfile

@@ -0,0 +1,74 @@
+FROM python:3.9-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies for multi-format document processing
+RUN apt-get update && apt-get install -y \
+    curl \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    libgomp1 \
+    libgstreamer1.0-0 \
+    libavcodec-dev \
+    libavformat-dev \
+    libswscale-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create cache directories with proper permissions before copying files
+RUN mkdir -p /app/.cache/huggingface && \
+    mkdir -p /app/.cache/sentence_transformers && \
+    chmod -R 777 /app/.cache
+
+# Set environment variables for HuggingFace cache (before installing packages)
+ENV HF_HOME=/app/.cache/huggingface
+ENV TRANSFORMERS_CACHE=/app/.cache/huggingface
+ENV SENTENCE_TRANSFORMERS_HOME=/app/.cache/sentence_transformers
+ENV HF_HUB_CACHE=/app/.cache/huggingface
+
+# Copy requirements first for better Docker layer caching
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application files AND folder structure
+COPY *.py ./
+COPY *.sh ./
+COPY requirements.txt ./
+COPY *.md ./
+COPY api/ ./api/
+COPY config/ ./config/
+COPY LLM/ ./LLM/
+COPY RAG/ ./RAG/
+COPY logger/ ./logger/
+COPY preprocessing/ ./preprocessing/
+
+# Set up directories and permissions during build (when we have root access)
+RUN mkdir -p /app/.cache/huggingface && \
+    mkdir -p /app/.cache/sentence_transformers && \
+    chmod -R 777 /app/.cache && \
+    chmod +x startup.sh && \
+    if [ -d "RAG/rag_embeddings" ]; then \
+        find RAG/rag_embeddings -name "*.lock" -delete 2>/dev/null || true; \
+        find RAG/rag_embeddings -name "*.db-shm" -delete 2>/dev/null || true; \
+        find RAG/rag_embeddings -name "*.db-wal" -delete 2>/dev/null || true; \
+        chmod -R 755 RAG/rag_embeddings; \
+    fi
+
+# Expose port
+EXPOSE 7860
+
+# Set environment variables
+ENV HOST=0.0.0.0
+ENV PORT=7860
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=30s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:7860/health || exit 1
+
+# Run the application
+CMD ["bash", "startup.sh"]

LLM/__init__.py

@@ -0,0 +1,13 @@
+# LLM Handler Package
+
+from .llm_handler import llm_handler
+from .tabular_answer import get_answer_for_tabluar
+from .image_answerer import get_answer_for_image
+from .one_shotter import get_oneshot_answer
+
+__all__ = [
+    'llm_handler',
+    'get_answer_for_tabluar',
+    'get_answer_for_image', 
+    'get_oneshot_answer'
+]

LLM/image_answerer.py

@@ -0,0 +1,136 @@
+import os
+import requests
+import google.generativeai as genai
+from PIL import Image
+from io import BytesIO
+from typing import List, Union
+import logging
+
+from dotenv import load_dotenv
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+load_dotenv()
+
+# Configure Gemini API for image processing
+genai.configure(api_key=os.getenv("GEMINI_API_KEY_IMAGE"))
+
+def load_image(image_source: str) -> Image.Image:
+    """Load image from a URL or local path."""
+    try:
+        if image_source.startswith("http://") or image_source.startswith("https://"):
+            logger.info(f"Loading image from URL: {image_source}")
+            response = requests.get(image_source, timeout=30)
+            response.raise_for_status()
+            return Image.open(BytesIO(response.content)).convert("RGB")
+        elif os.path.isfile(image_source):
+            logger.info(f"Loading image from file: {image_source}")
+            return Image.open(image_source).convert("RGB")
+        else:
+            raise ValueError("Invalid image source: must be a valid URL or file path")
+    except Exception as e:
+        logger.error(f"Failed to load image from {image_source}: {e}")
+        raise RuntimeError(f"Failed to load image: {e}")
+
+def get_answer_for_image(image_source: str, questions: List[str], retries: int = 3) -> List[str]:
+    """Ask questions about an image using Gemini Vision model."""
+    try:
+        logger.info(f"Processing image with {len(questions)} questions")
+        image = load_image(image_source)
+        
+        prompt = """
+Answer the following questions about the image. Give the answers in the same order as the questions. 
+Answers should be descriptive and detailed. Give one answer per line with numbering as "1. 2. 3. ..".
+
+Example answer format:
+1. Answer 1, Explanation
+2. Answer 2, Explanation
+3. Answer 3, Explanation
+
+Questions: 
+"""
+        prompt += "\n".join(f"{i+1}. {q}" for i, q in enumerate(questions))
+
+        model = genai.GenerativeModel("gemini-1.5-flash")
+        
+        for attempt in range(retries):
+            try:
+                logger.info(f"Attempt {attempt + 1} of {retries} to get response from Gemini")
+                response = model.generate_content(
+                    [prompt, image],
+                    generation_config=genai.types.GenerationConfig(
+                        temperature=0.4,
+                        max_output_tokens=2048
+                    )
+                )
+                raw_text = response.text.strip()
+                logger.info(f"Received response from Gemini: {len(raw_text)} characters")
+                
+                answers = extract_ordered_answers(raw_text, len(questions))
+                if len(answers) == len(questions):
+                    logger.info(f"Successfully extracted {len(answers)} answers")
+                    return answers
+                else:
+                    logger.warning(f"Expected {len(questions)} answers, got {len(answers)}")
+                    
+            except Exception as e:
+                logger.error(f"Attempt {attempt + 1} failed: {e}")
+                if attempt == retries - 1:
+                    raise RuntimeError(f"Failed after {retries} attempts: {e}")
+
+        raise RuntimeError("Failed to get valid response from Gemini.")
+        
+    except Exception as e:
+        logger.error(f"Error in get_answer_for_image: {e}")
+        raise
+
+def extract_ordered_answers(raw_text: str, expected_count: int) -> List[str]:
+    """Parse the raw Gemini output into a clean list of answers."""
+    import re
+    
+    logger.debug(f"Extracting {expected_count} answers from raw text")
+    lines = raw_text.splitlines()
+    answers = []
+    
+    for line in lines:
+        # Match numbered lines: "1. Answer", "1) Answer", "1 - Answer", etc.
+        match = re.match(r"^\s*(\d+)[\).\s-]*\s*(.+)", line)
+        if match:
+            answer_text = match.group(2).strip()
+            if answer_text:  # Only add non-empty answers
+                answers.append(answer_text)
+    
+    # Fallback: if numbering failed, use plain lines
+    if len(answers) < expected_count:
+        logger.warning("Numbered extraction failed, using fallback method")
+        answers = [line.strip() for line in lines if line.strip()]
+    
+    # Return exactly the expected number of answers
+    result = answers[:expected_count]
+    
+    # If we still don't have enough answers, pad with error messages
+    while len(result) < expected_count:
+        result.append("Unable to extract answer from image")
+    
+    logger.info(f"Extracted {len(result)} answers")
+    return result
+
+def process_image_query(image_path: str, query: str) -> str:
+    """Process a single query about an image."""
+    try:
+        questions = [query]
+        answers = get_answer_for_image(image_path, questions)
+        return answers[0] if answers else "Unable to process image query"
+    except Exception as e:
+        logger.error(f"Error processing image query: {e}")
+        return f"Error processing image: {str(e)}"
+
+def process_multiple_image_queries(image_path: str, queries: List[str]) -> List[str]:
+    """Process multiple queries about an image."""
+    try:
+        return get_answer_for_image(image_path, queries)
+    except Exception as e:
+        logger.error(f"Error processing multiple image queries: {e}")
+        return [f"Error processing image: {str(e)}"] * len(queries)

LLM/llm_handler.py

@@ -0,0 +1,235 @@
+"""
+Multi-LLM Handler with failover support
+Uses Groq, Gemini, and OpenAI with automatic failover for reliability
+"""
+
+import asyncio
+import re
+import time
+from typing import Optional, Dict, Any, List
+import os
+import requests
+import google.generativeai as genai
+import openai
+from dotenv import load_dotenv
+from config.config import get_provider_configs
+
+load_dotenv()
+
+class MultiLLMHandler:
+    """Multi-LLM handler with automatic failover across providers."""
+    
+    def __init__(self):
+        """Initialize the multi-LLM handler with all available providers."""
+        self.providers = get_provider_configs()
+        self.current_provider = None
+        self.current_config = None
+
+        # Initialize the first available provider (prefer Gemini/OpenAI for general RAG)
+        self._initialize_provider()
+
+        print(f"✅ Initialized Multi-LLM Handler with {self.provider.upper()}: {self.model_name}")
+
+    def _initialize_provider(self):
+        """Initialize the first available provider."""
+        # Prefer Gemini first for general text tasks
+        if self.providers["gemini"]:
+            self.current_provider = "gemini"
+            self.current_config = self.providers["gemini"][0]
+            genai.configure(api_key=self.current_config["api_key"])
+        # Then OpenAI
+        elif self.providers["openai"]:
+            self.current_provider = "openai"
+            self.current_config = self.providers["openai"][0]
+            openai.api_key = self.current_config["api_key"]
+        # Finally Groq
+        elif self.providers["groq"]:
+            self.current_provider = "groq"
+            self.current_config = self.providers["groq"][0]
+        else:
+            raise ValueError("No LLM providers available with valid API keys")
+
+    @property
+    def provider(self):
+        """Get current provider name."""
+        return self.current_provider
+
+    @property
+    def model_name(self):
+        """Get current model name."""
+        return self.current_config["model"] if self.current_config else "unknown"
+
+    async def _call_groq(self, prompt: str, temperature: float, max_tokens: int) -> str:
+        """Call Groq API."""
+        headers = {
+            "Authorization": f"Bearer {self.current_config['api_key']}",
+            "Content-Type": "application/json"
+        }
+        
+        data = {
+            "model": self.current_config["model"],
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": temperature,
+            "max_tokens": max_tokens
+        }
+
+        # Hide reasoning tokens (e.g., <think>) for Qwen reasoning models
+        try:
+            model_name = (self.current_config.get("model") or "").lower()
+            if "qwen" in model_name:
+                # Per request, use the chat completion parameter to hide reasoning content
+                data["reasoning_effort"] = "hidden"
+        except Exception:
+            # Be resilient if config shape changes
+            pass
+        
+        response = requests.post(
+            "https://api.groq.com/openai/v1/chat/completions", 
+            headers=headers, 
+            json=data, 
+            timeout=30
+        )
+        response.raise_for_status()
+        
+        result = response.json()
+        text = result["choices"][0]["message"]["content"].strip()
+        # Safety net: strip any <think>...</think> blocks if present
+        try:
+            model_name = (self.current_config.get("model") or "").lower()
+            if "qwen" in model_name and "<think>" in text.lower():
+                text = re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.IGNORECASE).strip()
+        except Exception:
+            pass
+        return text
+
+    async def _call_gemini(self, prompt: str, temperature: float, max_tokens: int) -> str:
+        """Call Gemini API."""
+        model = genai.GenerativeModel(self.current_config["model"])
+        
+        generation_config = genai.types.GenerationConfig(
+            temperature=temperature,
+            max_output_tokens=max_tokens
+        )
+        
+        response = await asyncio.to_thread(
+            model.generate_content,
+            prompt,
+            generation_config=generation_config
+        )
+        return response.text.strip()
+
+    async def _call_openai(self, prompt: str, temperature: float, max_tokens: int) -> str:
+        """Call OpenAI API."""
+        response = await asyncio.to_thread(
+            openai.ChatCompletion.create,
+            model=self.current_config["model"],
+            messages=[{"role": "user", "content": prompt}],
+            temperature=temperature,
+            max_tokens=max_tokens
+        )
+        return response.choices[0].message.content.strip()
+
+    async def _try_with_failover(self, prompt: str, temperature: float, max_tokens: int) -> str:
+        """Try to generate text with automatic failover."""
+        # Get all available providers in order
+        provider_order = []
+        # Prefer Gemini -> OpenAI -> Groq for general text
+        if self.providers["gemini"]:
+            provider_order.extend([("gemini", config) for config in self.providers["gemini"]])
+        if self.providers["openai"]:
+            provider_order.extend([("openai", config) for config in self.providers["openai"]])
+        if self.providers["groq"]:
+            provider_order.extend([("groq", config) for config in self.providers["groq"]])
+        
+        last_error = None
+        
+        for provider_name, config in provider_order:
+            try:
+                # Set current provider
+                old_provider = self.current_provider
+                old_config = self.current_config
+                
+                self.current_provider = provider_name
+                self.current_config = config
+                
+                # Configure API if needed
+                if provider_name == "gemini":
+                    genai.configure(api_key=config["api_key"])
+                elif provider_name == "openai":
+                    openai.api_key = config["api_key"]
+                
+                # Try the API call
+                if provider_name == "groq":
+                    return await self._call_groq(prompt, temperature, max_tokens)
+                elif provider_name == "gemini":
+                    return await self._call_gemini(prompt, temperature, max_tokens)
+                elif provider_name == "openai":
+                    return await self._call_openai(prompt, temperature, max_tokens)
+                    
+            except Exception as e:
+                print(f"⚠️ {provider_name.upper()} ({config['name']}) failed: {str(e)}")
+                last_error = e
+                
+                # Restore previous provider
+                self.current_provider = old_provider
+                self.current_config = old_config
+                continue
+        
+        # If all providers failed
+        raise RuntimeError(f"All LLM providers failed. Last error: {last_error}")
+
+    async def generate_text(self, 
+                          prompt: Optional[str] = None,
+                          system_prompt: Optional[str] = None,
+                          user_prompt: Optional[str] = None,
+                          temperature: Optional[float] = 0.4,
+                          max_tokens: Optional[int] = 1200) -> str:
+        """Generate text using multi-LLM with failover."""
+        # Handle both single prompt and system/user prompt formats
+        if prompt:
+            final_prompt = prompt
+        elif system_prompt and user_prompt:
+            final_prompt = f"{system_prompt}\n\n{user_prompt}"
+        elif user_prompt:
+            final_prompt = user_prompt
+        else:
+            raise ValueError("Must provide either 'prompt' or 'user_prompt'")
+        
+        return await self._try_with_failover(
+            final_prompt, 
+            temperature or 0.4, 
+            max_tokens or 1200
+        )
+
+    async def generate_simple(self, 
+                            prompt: str, 
+                            temperature: Optional[float] = 0.4,
+                            max_tokens: Optional[int] = 1200) -> str:
+        """Simple text generation (alias for generate_text for compatibility)."""
+        return await self.generate_text(prompt=prompt, temperature=temperature, max_tokens=max_tokens)
+
+    def get_provider_info(self) -> Dict[str, Any]:
+        """Get information about the current provider."""
+        return {
+            "provider": self.current_provider,
+            "model": self.model_name,
+            "config_name": self.current_config["name"] if self.current_config else "none",
+            "available_providers": {
+                "groq": len(self.providers["groq"]),
+                "gemini": len(self.providers["gemini"]),
+                "openai": len(self.providers["openai"])
+            }
+        }
+
+    async def test_connection(self) -> bool:
+        """Test the connection to the current LLM provider."""
+        try:
+            test_prompt = "Say 'Hello' if you can read this."
+            response = await self.generate_simple(test_prompt, temperature=0.1, max_tokens=10)
+            return "hello" in response.lower()
+        except Exception as e:
+            print(f"❌ Connection test failed: {str(e)}")
+            return False
+
+# Create a global instance
+llm_handler = MultiLLMHandler()

LLM/one_shotter.py

@@ -0,0 +1,218 @@
+import re
+import asyncio
+from typing import List, Dict
+from urllib.parse import urlparse
+import httpx
+from bs4 import BeautifulSoup
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# Import our multi-LLM handler
+from LLM.llm_handler import llm_handler
+
+# URL extraction pattern (same as ShastraDocs)
+URL_PATTERN = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
+
+def extract_urls_from_text(text: str) -> List[str]:
+    urls = URL_PATTERN.findall(text or "")
+    seen = set()
+    clean_urls = []
+    for url in urls:
+        clean_url = url.rstrip('.,;:!?)')
+        if clean_url and clean_url not in seen and validate_url(clean_url):
+            seen.add(clean_url)
+            clean_urls.append(clean_url)
+    return clean_urls
+
+def validate_url(url: str) -> bool:
+    try:
+        result = urlparse(url)
+        return bool(result.scheme and result.netloc)
+    except Exception:
+        return False
+
+async def scrape_url(url: str, max_chars: int = 4000) -> Dict[str, str]:
+    """Async URL scraping using httpx + BeautifulSoup (FastAPI-friendly)."""
+    try:
+        timeout = httpx.Timeout(20.0)
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+        async with httpx.AsyncClient(timeout=timeout, headers=headers, follow_redirects=True) as client:
+            resp = await client.get(url)
+            resp.raise_for_status()
+            soup = BeautifulSoup(resp.content, 'html.parser')
+            for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+                tag.decompose()
+            text_content = soup.get_text(separator=' ', strip=True)
+            cleaned = ' '.join(text_content.split())
+            if len(cleaned) > max_chars:
+                cleaned = cleaned[:max_chars] + "..."
+            return {
+                'url': url,
+                'content': cleaned,
+                'status': 'success',
+                'length': len(cleaned),
+                'title': soup.title.string if soup.title else 'No title'
+            }
+    except httpx.TimeoutException:
+        return {'url': url, 'content': 'Timeout error', 'status': 'timeout', 'length': 0, 'title': 'Timeout'}
+    except Exception as e:
+        return {'url': url, 'content': f'Error: {str(e)[:100]}', 'status': 'error', 'length': 0, 'title': 'Error'}
+
+async def scrape_urls(urls: List[str], max_chars: int = 4000) -> List[Dict[str, str]]:
+    if not urls:
+        return []
+    sem = asyncio.Semaphore(5)
+    async def _scrape(u):
+        async with sem:
+            return await scrape_url(u, max_chars)
+    results = await asyncio.gather(*[_scrape(u) for u in urls], return_exceptions=True)
+    final = []
+    for i, r in enumerate(results):
+        if isinstance(r, Exception):
+            final.append({'url': urls[i], 'content': f'Exception: {str(r)[:100]}', 'status': 'exception', 'length': 0, 'title': 'Exception'})
+        else:
+            final.append(r)
+    return final
+
+def build_additional_content(scrapes: List[Dict[str, str]]) -> str:
+    parts = []
+    for r in scrapes:
+        if r.get('status') == 'success' and r.get('length', 0) > 50:
+            parts.append("\n" + "="*50)
+            parts.append(f"SOURCE: Additional Source")
+            parts.append(f"URL: {r.get('url','')}")
+            parts.append(f"TITLE: {r.get('title','No title')}")
+            parts.append("-"*30 + " CONTENT " + "-"*30)
+            parts.append(r.get('content',''))
+            parts.append("="*50)
+    return "\n".join(parts)
+
+def parse_numbered_answers(text: str, expected_count: int) -> List[str]:
+    """Parse numbered answers, with sane fallbacks."""
+    pattern = re.compile(r'^\s*(\d+)[\).\-]\s*(.+)$', re.MULTILINE)
+    matches = pattern.findall(text or "")
+    result: Dict[int, str] = {}
+    for num_str, answer in matches:
+        try:
+            num = int(num_str)
+            if 1 <= num <= expected_count:
+                clean_answer = re.sub(r'\s+', ' ', answer).strip()
+                if clean_answer:
+                    result[num] = clean_answer
+        except Exception:
+            continue
+    answers: List[str] = []
+    for i in range(1, expected_count + 1):
+        answers.append(result.get(i, f"Unable to find answer for question {i}"))
+    return answers
+
+def parse_answers_from_json(raw: str, expected_count: int) -> List[str]:
+    import json, re
+    # Try direct JSON
+    try:
+        obj = json.loads(raw)
+        if isinstance(obj, dict) and isinstance(obj.get('answers'), list):
+            out = [str(x).strip() for x in obj['answers']][:expected_count]
+            while len(out) < expected_count:
+                out.append(f"Unable to find answer for question {len(out)+1}")
+            return out
+    except Exception:
+        pass
+    # Try to extract JSON fragment
+    m = re.search(r'\{[^\{\}]*"answers"[^\{\}]*\}', raw or "", re.DOTALL)
+    if m:
+        try:
+            obj = json.loads(m.group(0))
+            if isinstance(obj, dict) and isinstance(obj.get('answers'), list):
+                out = [str(x).strip() for x in obj['answers']][:expected_count]
+                while len(out) < expected_count:
+                    out.append(f"Unable to find answer for question {len(out)+1}")
+                return out
+        except Exception:
+            pass
+    # Fallback to numbered parsing
+    return parse_numbered_answers(raw or "", expected_count)
+
+async def get_oneshot_answer(content: str, questions: List[str]) -> List[str]:
+    """
+    Enhanced oneshot QA flow with ShastraDocs-style URL extraction and scraping.
+    - Extract URLs from content and questions
+    - Scrape relevant pages
+    - Merge additional content and feed to LLM
+    - Return per-question answers
+    """
+    if not questions:
+        return []
+
+    try:
+        # Build numbered questions
+        numbered_questions = "\n".join([f"{i+1}. {q}" for i, q in enumerate(questions)])
+
+        # Find URLs from content and questions
+        combined = (content or "") + "\n" + "\n".join(questions or [])
+        found_urls = extract_urls_from_text(combined)
+
+        # Special case: content starts with URL marker
+        if content.startswith("URL for Context:"):
+            only_url = content.replace("URL for Context:", "").strip()
+            if validate_url(only_url):
+                if only_url not in found_urls:
+                    found_urls.insert(0, only_url)
+
+        # Scrape URLs if any
+        additional_content = ""
+        if found_urls:
+            print(f"🚀 Scraping {len(found_urls)} URL(s) for additional context...")
+            scrape_results = await scrape_urls(found_urls, max_chars=4000)
+            additional_content = build_additional_content(scrape_results)
+            print(f"📄 Additional content length: {len(additional_content)}")
+
+        # Merge final context
+        if additional_content:
+            final_context = (content or "") + "\n\nADDITIONAL INFORMATION FROM SCRAPED SOURCES:\n" + additional_content
+        else:
+            final_context = content or ""
+
+        print(f"📊 Final context length: {len(final_context)}")
+
+        # Prompts (ask for JSON answers to improve parsing)
+        system_prompt = (
+            "You are an expert assistant. Read ALL provided context (including any 'ADDITIONAL INFORMATION FROM\n"
+            "SCRAPED SOURCES') and answer the questions comprehensively. If info is missing, say so."
+        )
+
+        user_prompt = f"""FULL CONTEXT:
+{final_context[:8000]}{"..." if len(final_context) > 8000 else ""}
+
+QUESTIONS:
+{numbered_questions}
+
+Respond in this EXACT JSON format:
+{{
+    "answers": [
+        "<Answer to question 1>",
+        "<Answer to question 2>",
+        "<Answer to question 3>"
+    ]
+}}"""
+
+        print(f"🤖 Using {llm_handler.provider.upper()} model: {llm_handler.model_name}")
+        raw = await llm_handler.generate_text(
+            system_prompt=system_prompt,
+            user_prompt=user_prompt,
+            temperature=0.4,
+            max_tokens=1800
+        )
+
+        print(f"🔄 LLM response length: {len(raw) if raw else 0}")
+        answers = parse_answers_from_json(raw, len(questions))
+        print(f"✅ Parsed {len(answers)} answers")
+        return answers
+
+    except Exception as e:
+        print(f"❌ Error in oneshot answer generation: {str(e)}")
+        return [f"Error processing question: {str(e)}" for _ in questions]

LLM/tabular_answer.py

@@ -0,0 +1,128 @@
+import os
+import re
+import math
+from typing import List
+from langchain_core.messages import SystemMessage, HumanMessage
+from langchain_groq import ChatGroq
+from dotenv import load_dotenv
+
+load_dotenv()
+TABULAR_VERBOSE = os.environ.get("TABULAR_VERBOSE", "0") in ("1", "true", "True", "yes", "YES")
+
+# Initialize Groq LLM for tabular data using specialized API key
+TABULAR_MODEL = os.environ.get("GROQ_TABULAR_MODEL", os.environ.get("GROQ_MODEL_TABULAR", "qwen/qwen3-32b"))
+GROQ_LLM = ChatGroq(
+    groq_api_key=os.environ.get("GROQ_API_KEY_TABULAR", os.environ.get("GROQ_API_KEY")),
+    model_name=TABULAR_MODEL
+)
+
+def get_answer_for_tabluar(
+    data: str,
+    questions: List[str],
+    batch_size: int = 10,
+    verbose: bool = False
+) -> List[str]:
+    """
+    Query Groq LLM for tabular data analysis, handling batches and preserving order of answers.
+
+    Args:
+        data (str): Tabular context in markdown or plain-text.
+        questions (List[str]): List of questions to ask.
+        batch_size (int): Max number of questions per batch.
+        verbose (bool): If True, print raw LLM responses.
+
+    Returns:
+        List[str]: Ordered list of answers corresponding to input questions.
+    """
+
+    def parse_numbered_answers(text: str, expected: int) -> List[str]:
+        """
+        Parse answers from a numbered list format ('1.', '2.', etc.)
+        Use non-greedy capture with lookahead to stop at the next number or end.
+        """
+        pattern = re.compile(r"^\s*(\d{1,2})[\.)\-]\s*(.*?)(?=\n\s*\d{1,2}[\.)\-]\s*|$)", re.MULTILINE | re.DOTALL)
+        matches = pattern.findall(text)
+
+        result = {}
+        for num_str, answer in matches:
+            try:
+                num = int(num_str)
+            except ValueError:
+                continue
+            if 1 <= num <= expected:
+                clean_answer = re.sub(r'\s+', ' ', answer).strip()
+                result[num] = clean_answer
+
+        # If no structured matches, fall back to line-based heuristic
+        if not result:
+            lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()]
+            for i in range(min(expected, len(lines))):
+                result[i + 1] = lines[i]
+
+        # Build fixed-length list
+        answers = []
+        for i in range(1, expected + 1):
+            answers.append(result.get(i, f"Unable to answer question {i}"))
+
+        return answers
+
+    if not questions:
+        return []
+
+    # Process questions in batches
+    all_answers = []
+    total_batches = math.ceil(len(questions) / batch_size)
+    
+    for batch_idx in range(total_batches):
+        start = batch_idx * batch_size
+        end = min(start + batch_size, len(questions))
+        batch_questions = questions[start:end]
+        
+        print(f"Processing batch {batch_idx + 1}/{total_batches} ({len(batch_questions)} questions)")
+        
+        # Create numbered question list
+        numbered_questions = "\\n".join([f"{i+1}. {q}" for i, q in enumerate(batch_questions)])
+        
+        # Create prompt
+        system_prompt = """You are an expert data analyst. Analyze the provided tabular data and answer the questions accurately.
+        
+Instructions:
+- Answer each question based ONLY on the data provided
+- If data is insufficient, state "Information not available in the provided data"
+- Provide clear, concise answers
+- Format your response as a numbered list (1., 2., 3., etc.)
+- Do not add explanations unless specifically asked"""
+
+        user_prompt = f"""Data:
+{data}
+
+Questions:
+{numbered_questions}
+
+Please provide numbered answers (1., 2., 3., etc.) for each question."""
+
+        try:
+            # Create messages
+            messages = [
+                SystemMessage(content=system_prompt),
+                HumanMessage(content=user_prompt)
+            ]
+            
+            # Get response from LLM
+            response = GROQ_LLM.invoke(messages)
+            raw_response = response.content or ""
+            
+            if verbose or TABULAR_VERBOSE:
+                print(f"🟢 Raw LLM Response (batch {batch_idx + 1}):\n{raw_response[:1200]}\n--- END RAW ---")
+            
+            # Parse the response
+            batch_answers = parse_numbered_answers(raw_response, len(batch_questions))
+            all_answers.extend(batch_answers)
+            
+        except Exception as e:
+            print(f"Error processing batch {batch_idx + 1}: {str(e)}")
+            # Add error answers for this batch
+            error_answers = [f"Error processing question: {str(e)}" for _ in batch_questions]
+            all_answers.extend(error_answers)
+    
+    return all_answers

RAG/__init__.py

@@ -0,0 +1 @@
+# RAG Package

RAG/advanced_rag_processor.py

@@ -0,0 +1,169 @@
+"""
+Advanced RAG Processor - Modular Version
+Orchestrates all RAG components for document question answering.
+Version: 3.0 - Modular Architecture
+"""
+
+import time
+from typing import Dict, Tuple
+from pathlib import Path
+
+# Import all modular components
+from RAG.rag_modules.query_expansion import QueryExpansionManager
+from RAG.rag_modules.embedding_manager import EmbeddingManager
+from RAG.rag_modules.search_manager import SearchManager
+from RAG.rag_modules.reranking_manager import RerankingManager
+from RAG.rag_modules.context_manager import ContextManager
+from RAG.rag_modules.answer_generator import AnswerGenerator
+
+from LLM.llm_handler import llm_handler
+from config.config import OUTPUT_DIR, TOP_K
+
+
+class AdvancedRAGProcessor:
+    """
+    Advanced RAG processor with modular architecture for better maintainability.
+    Orchestrates query expansion, hybrid search, reranking, and answer generation.
+    """
+    
+    def __init__(self):
+        """Initialize the advanced RAG processor with all modules."""
+        self.base_db_path = Path(OUTPUT_DIR)
+        
+        # Initialize all managers
+        print("🚀 Initializing Advanced RAG Processor (Modular)...")
+        
+        # Core components
+        self.embedding_manager = EmbeddingManager()
+        self.query_expansion_manager = QueryExpansionManager()
+        self.search_manager = SearchManager(self.embedding_manager)
+        self.reranking_manager = RerankingManager()
+        self.context_manager = ContextManager()
+        self.answer_generator = AnswerGenerator()
+        
+        # Keep reference to LLM handler for info
+        self.llm_handler = llm_handler
+        
+        print(f"✅ Advanced RAG Processor initialized with {self.llm_handler.provider.upper()} LLM")
+        print("📦 All modules loaded successfully:")
+        print("   🔄 Query Expansion Manager")
+        print("   🧠 Embedding Manager")
+        print("   🔍 Search Manager (Hybrid)")
+        print("   🎯 Reranking Manager")
+        print("   📝 Context Manager")
+        print("   💬 Answer Generator")
+    
+    async def answer_question(self, question: str, doc_id: str, logger=None, request_id: str = None) -> Tuple[str, Dict[str, float]]:
+        """
+        Answer a question using advanced RAG techniques with detailed timing.
+        
+        Args:
+            question: The question to answer
+            doc_id: Document ID to search in
+            logger: Optional logger for tracking
+            request_id: Optional request ID for logging
+            
+        Returns:
+            Tuple of (answer, timing_breakdown)
+        """
+        timings = {}
+        overall_start = time.time()
+        
+        try:
+            # Check if collection exists
+            collection_name = f"{doc_id}_collection"
+            try:
+                client = self.search_manager.get_qdrant_client(doc_id)
+                collection_info = client.get_collection(collection_name)
+            except Exception:
+                return "I don't have information about this document. Please ensure the document has been processed.", timings
+            
+            print(f"🚀 Advanced RAG processing for: {question[:100]}...")
+            
+            # Step 1: Query Expansion
+            step_start = time.time()
+            expanded_queries = await self.query_expansion_manager.expand_query(question)
+            expansion_time = time.time() - step_start
+            timings['query_expansion'] = expansion_time
+            if logger and request_id:
+                logger.log_pipeline_stage(request_id, "query_expansion", expansion_time)
+            
+            # Step 2: Hybrid Search with Fusion
+            step_start = time.time()
+            search_results = await self.search_manager.hybrid_search(expanded_queries, doc_id, TOP_K)
+            search_time = time.time() - step_start
+            timings['hybrid_search'] = search_time
+            if logger and request_id:
+                logger.log_pipeline_stage(request_id, "hybrid_search", search_time)
+            
+            if not search_results:
+                return "I couldn't find relevant information to answer your question.", timings
+            
+            # Step 3: Reranking
+            step_start = time.time()
+            reranked_results = await self.reranking_manager.rerank_results(question, search_results)
+            rerank_time = time.time() - step_start
+            timings['reranking'] = rerank_time
+            if logger and request_id:
+                logger.log_pipeline_stage(request_id, "reranking", rerank_time)
+            
+            # Step 4: Multi-perspective Context Creation
+            step_start = time.time()
+            context = self.context_manager.create_enhanced_context(question, reranked_results)
+            context_time = time.time() - step_start
+            timings['context_creation'] = context_time
+            if logger and request_id:
+                logger.log_pipeline_stage(request_id, "context_creation", context_time)
+
+            # Step 5: Enhanced Answer Generation
+            step_start = time.time()
+            answer = await self.answer_generator.generate_enhanced_answer(question, context, expanded_queries)
+            generation_time = time.time() - step_start
+            timings['llm_generation'] = generation_time
+            if logger and request_id:
+                logger.log_pipeline_stage(request_id, "llm_generation", generation_time)
+
+            # Calculate total time
+            total_time = time.time() - overall_start
+            timings['total_pipeline'] = total_time
+
+            print(f"\n✅ Advanced RAG processing completed in {total_time:.4f}s")
+            print(f"   🔍 Query expansion: {expansion_time:.4f}s")
+            print(f"   🔎 Hybrid search: {search_time:.4f}s") 
+            print(f"   🎯 Reranking: {rerank_time:.4f}s")
+            print(f"   📝 Context creation: {context_time:.4f}s")
+            print(f"   💬 LLM generation: {generation_time:.4f}s")
+
+            return answer, timings
+            
+        except Exception as e:
+            error_time = time.time() - overall_start
+            timings['error_time'] = error_time
+            print(f"❌ Error in advanced RAG processing: {str(e)}")
+            return f"I encountered an error while processing your question: {str(e)}", timings
+    
+    def cleanup(self):
+        """Cleanup all manager resources."""
+        print("🧹 Cleaning up Advanced RAG processor resources...")
+        
+        # Cleanup search manager (which has the most resources)
+        self.search_manager.cleanup()
+        
+        print("✅ Advanced RAG cleanup completed")
+    
+    def get_system_info(self) -> Dict:
+        """Get information about the RAG system."""
+        return {
+            "version": "3.0 - Modular",
+            "llm_provider": self.llm_handler.provider,
+            "llm_model": self.llm_handler.model_name,
+            "modules": [
+                "QueryExpansionManager",
+                "EmbeddingManager", 
+                "SearchManager",
+                "RerankingManager",
+                "ContextManager",
+                "AnswerGenerator"
+            ],
+            "base_db_path": str(self.base_db_path)
+        }

RAG/rag_modules/__init__.py

@@ -0,0 +1 @@
+# RAG Modules Package

RAG/rag_modules/answer_generator.py

@@ -0,0 +1,97 @@
+"""
+Answer Generation Module for Advanced RAG
+Handles LLM-based answer generation with enhanced prompting.
+"""
+
+from typing import List
+from LLM.llm_handler import llm_handler
+from config.config import TEMPERATURE, MAX_TOKENS
+
+
+class AnswerGenerator:
+    """Manages answer generation using LLM."""
+    
+    def __init__(self):
+        """Initialize the answer generator."""
+        self.llm_handler = llm_handler
+        print("✅ Answer Generator initialized")
+    
+    async def generate_enhanced_answer(self, original_question: str, context: str, expanded_queries: List[str]) -> str:
+        """Generate enhanced answer using the original question with retrieved context."""
+        
+        # Use only the original question for LLM generation
+        query_context = f"Question: {original_question}"
+        
+        system_prompt = """
+
+You are an expert AI assistant specializing in document analysis and policy-related question answering. You have access to relevant document excerpts and must respond only based on this information. You are designed specifically for analyzing official documents and answering user queries related to them.
+
+STRICT RULES AND RESPONSE CONDITIONS:
+
+    Irrelevant/Out-of-Scope Queries (e.g., programming help, general product info, coding tasks):
+    Respond EXACTLY:
+
+        "I cannot help with that. I am designed only to answer queries related to the provided document excerpts."
+
+    Illegal or Prohibited Requests (e.g., forgery, fraud, bypassing regulations):
+    Respond CLEARLY that the request is illegal. Example format:
+
+        "This request is illegal and cannot be supported. According to the applicable regulations in the document, [explain why it's illegal if mentioned]. Engaging in such activity may lead to legal consequences."
+        If illegality is not explicitly in the documents, use:
+        "This request involves illegal activity and is against policy. I cannot assist with this."
+
+    Nonexistent Concepts, Schemes, or Entities:
+    Respond by stating the concept does not exist and offer clarification by pointing to related valid information. Example:
+
+        "There is no mention of such a scheme in the document. However, the following related schemes are described: [summarize relevant ones]."
+
+    Valid Topics with Missing or Incomplete Information:
+    Respond that the exact answer is unavailable, then provide all related details and recommend official contact. Example:
+
+        "The exact information is not available in the provided document. However, here is what is relevant: [details]. For further clarification, you may contact: [official contact details if included in the document]."
+
+    Valid Questions Answerable from Document:
+    Provide a concise and accurate answer with clear reference to the document content. Also include any related notes that might aid understanding. Example:
+
+        "[Answer]. According to the policy document, [quote/summary from actual document content]."
+
+GENERAL ANSWERING RULES:
+
+    Use ONLY the provided document excerpts. Never use external knowledge.
+
+    Be concise: 5-6 sentences per answer, with all the details available for that particular query.
+
+    Start directly with the answer. Do not restate or rephrase the question.
+
+    Never speculate or elaborate beyond what is explicitly stated.
+
+    When referencing information, mention "according to the document" or "as stated in the policy" rather than using internal labels like "Query X Doc Y".
+
+    Do not reference internal organizational labels like [Query 1 Doc 2] or [Relevance: X.XX] - these are for processing only.
+
+    Focus on the actual document content and policy information when providing answers.
+
+The user may phrase questions in various ways — always infer the intent, apply the rules above, and respond accordingly.
+
+"""
+
+        user_prompt = f"""{query_context}
+
+Document Excerpts:
+{context}
+
+Provide a comprehensive answer based on the document excerpts above:"""
+        
+        try:
+            answer = await self.llm_handler.generate_text(
+                system_prompt=system_prompt,
+                user_prompt=user_prompt,
+                temperature=TEMPERATURE,
+                max_tokens=MAX_TOKENS
+            )
+            
+            return answer.strip()
+            
+        except Exception as e:
+            print(f"❌ Error generating enhanced response with {self.llm_handler.provider.upper()}: {str(e)}")
+            return "I encountered an error while generating the response."

RAG/rag_modules/context_manager.py

@@ -0,0 +1,81 @@
+"""
+Context Management Module for Advanced RAG
+Handles context creation and management for LLM generation.
+"""
+
+from typing import List, Dict
+from collections import defaultdict
+from config.config import MAX_CONTEXT_LENGTH
+
+
+class ContextManager:
+    """Manages context creation for LLM generation."""
+    
+    def __init__(self):
+        """Initialize the context manager."""
+        print("✅ Context Manager initialized")
+    
+    def create_enhanced_context(self, question: str, results: List[Dict], max_length: int = MAX_CONTEXT_LENGTH) -> str:
+        """Create enhanced context ensuring each query contributes equally."""
+        # Group results by expanded query index
+        query_to_chunks = defaultdict(list)
+        for i, result in enumerate(results):
+            # Find the most relevant expanded query for this chunk
+            if 'contributing_queries' in result and result['contributing_queries']:
+                # Use the highest scoring contributing query
+                best_contrib = max(result['contributing_queries'], key=lambda cq: cq.get('semantic_score', cq.get('bm25_score', 0)))
+                query_idx = best_contrib['query_idx']
+            else:
+                query_idx = 0  # fallback to first query
+            query_to_chunks[query_idx].append((i, result))
+
+        # Sort chunks within each query by their relevance scores
+        for q_idx in query_to_chunks:
+            query_to_chunks[q_idx].sort(key=lambda x: x[1].get('rerank_score', x[1].get('final_score', x[1].get('score', 0))), reverse=True)
+
+        # Calculate chunks per query (should be 3 for each query with total budget = 9 and 3 queries)
+        num_queries = len(query_to_chunks)
+        if num_queries == 0:
+            return ""
+        
+        # Ensure each query contributes equally (round-robin with guaranteed slots)
+        context_parts = []
+        current_length = 0
+        added_chunks = set()
+        
+        # Calculate how many chunks each query should contribute
+        chunks_per_query = len(results) // num_queries if num_queries > 0 else len(results)
+        extra_chunks = len(results) % num_queries
+        
+        print(f"📊 Context Creation: {num_queries} queries, {chunks_per_query} chunks per query (+{extra_chunks} extra)")
+        
+        for q_idx in sorted(query_to_chunks.keys()):
+            # Determine how many chunks this query should contribute
+            query_chunk_limit = chunks_per_query + (1 if q_idx < extra_chunks else 0)
+            query_chunks_added = 0
+            
+            print(f"   Query {q_idx+1}: Adding up to {query_chunk_limit} chunks")
+            
+            for i, result in query_to_chunks[q_idx]:
+                if i not in added_chunks and query_chunks_added < query_chunk_limit:
+                    text = result['payload'].get('text', '')
+                    relevance_info = ""
+                    if 'rerank_score' in result:
+                        relevance_info = f" [Relevance: {result['rerank_score']:.2f}]"
+                    elif 'final_score' in result:
+                        relevance_info = f" [Score: {result['final_score']:.2f}]"
+                    doc_text = f"[Query {q_idx+1} Doc {len(added_chunks)+1}]{relevance_info}\n{text}\n"
+                    
+                    if current_length + len(doc_text) > max_length:
+                        print(f"   ⚠️ Context length limit reached at {current_length} chars")
+                        break
+                    
+                    context_parts.append(doc_text)
+                    current_length += len(doc_text)
+                    added_chunks.add(i)
+                    query_chunks_added += 1
+            
+            print(f"   Query {q_idx+1}: Added {query_chunks_added} chunks")
+        
+        print(f"📝 Final context: {len(added_chunks)} chunks, {current_length} chars")
+        return "\n".join(context_parts)

RAG/rag_modules/embedding_manager.py

@@ -0,0 +1,42 @@
+"""
+Embedding Management Module for Advanced RAG
+Handles text encoding and embedding operations.
+"""
+
+import asyncio
+from typing import List
+from sentence_transformers import SentenceTransformer
+from config.config import EMBEDDING_MODEL
+
+
+class EmbeddingManager:
+    """Manages text embeddings for RAG operations."""
+    
+    def __init__(self):
+        """Initialize the embedding manager."""
+        self.embedding_model = None
+        self._init_embedding_model()
+    
+    def _init_embedding_model(self):
+        """Initialize the embedding model."""
+        print(f"🔄 Loading embedding model: {EMBEDDING_MODEL}")
+        self.embedding_model = SentenceTransformer(EMBEDDING_MODEL)
+        print(f"✅ Embedding model loaded successfully")
+    
+    async def encode_query(self, query: str) -> List[float]:
+        """Encode a query into embeddings."""
+        def encode_sync():
+            embedding = self.embedding_model.encode([query], normalize_embeddings=True)
+            return embedding[0].astype("float32").tolist()
+        
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, encode_sync)
+    
+    async def encode_texts(self, texts: List[str]) -> List[List[float]]:
+        """Encode multiple texts into embeddings."""
+        def encode_sync():
+            embeddings = self.embedding_model.encode(texts, normalize_embeddings=True)
+            return [emb.astype("float32").tolist() for emb in embeddings]
+        
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, encode_sync)

RAG/rag_modules/query_expansion.py

@@ -0,0 +1,146 @@
+"""
+Query Expansion Module for Advanced RAG
+Now uses Groq's llama3-8b-8192 model directly for generating focused sub-queries.
+"""
+
+import re
+import time
+import asyncio
+from typing import List
+from groq import Groq
+from config.config import (
+    ENABLE_QUERY_EXPANSION,
+    QUERY_EXPANSION_COUNT,
+    GROQ_API_KEY_LITE,
+    GROQ_MODEL_LITE,
+)
+
+
+class QueryExpansionManager:
+    """Manages query expansion for better information retrieval."""
+    
+    def __init__(self):
+        """Initialize the query expansion manager with Groq client."""
+        # Initialize Groq client with the lite key and llama3-8b-8192 model
+        self.model = GROQ_MODEL_LITE or "llama3-8b-8192"
+        if not GROQ_API_KEY_LITE:
+            print("⚠️ GROQ_API_KEY_LITE is not set. Query expansion will fall back to original query.")
+            self.client = None
+        else:
+            self.client = Groq(api_key=GROQ_API_KEY_LITE)
+        print(f"✅ Query Expansion Manager initialized using Groq model: {self.model}")
+    
+    async def expand_query(self, original_query: str) -> List[str]:
+        """Break complex queries into focused parts for better information retrieval using Groq."""
+        if not ENABLE_QUERY_EXPANSION:
+            return [original_query]
+        if not self.client:
+            return [original_query]
+        
+        try:
+            expansion_prompt = f"""Analyze this question and break it down into exactly {QUERY_EXPANSION_COUNT} specific, focused sub-questions that can be searched independently in a document. Each sub-question should target a distinct piece of information or process.
+
+For complex questions with multiple parts, identify:
+1. Different processes or procedures mentioned
+2. Specific information requests (emails, contact details, forms, etc.)
+3. Different entities or subjects involved
+4. Sequential steps that might be documented separately
+
+Original question: {original_query}
+
+Break this into exactly {QUERY_EXPANSION_COUNT} focused search queries that target different aspects:
+
+Examples of good breakdown:
+- "What is the dental claim submission process?"
+- "How to update surname/name in policy records?"
+- "What are the company contact details and grievance email?"
+
+Provide only {QUERY_EXPANSION_COUNT} focused sub-questions, one per line, without numbering or additional formatting:"""
+
+            # Call Groq's chat completions in a thread to avoid blocking the event loop
+            response = await asyncio.to_thread(
+                self.client.chat.completions.create,
+                messages=[{"role": "user", "content": expansion_prompt}],
+                model=self.model,
+                temperature=0.3,
+                max_tokens=300,
+            )
+            
+            expanded_queries = []  # Start with empty list - don't include original
+
+            if response and response.choices:
+                content = response.choices[0].message.content if response.choices[0].message else ""
+                sub_queries = (content or "").strip().split('\n')
+                for query in sub_queries:
+                    if len(expanded_queries) >= QUERY_EXPANSION_COUNT:  # Stop when we have enough
+                        break
+                    query = query.strip()
+                    # Remove any numbering or bullet points that might be added
+                    query = re.sub(r'^[\d\.\-\*\s]+', '', query).strip()
+                    if query and len(query) > 10:
+                        expanded_queries.append(query)
+            
+            # If we don't have enough sub-queries, fall back to using the original
+            if len(expanded_queries) < QUERY_EXPANSION_COUNT:
+                expanded_queries = [original_query] * QUERY_EXPANSION_COUNT
+            
+            # Ensure we have exactly QUERY_EXPANSION_COUNT queries
+            final_queries = expanded_queries[:QUERY_EXPANSION_COUNT]
+            
+            print(f"🔄 Query broken down from 1 complex question to {len(final_queries)} focused sub-queries using Groq {self.model}")
+            print(f"📌 Original query will be used for final LLM generation only")
+            for i, q in enumerate(final_queries):
+                print(f"   Sub-query {i+1}: {q[:80]}...")
+            
+            return final_queries
+            
+        except Exception as e:
+            print(f"⚠️ Query expansion failed: {e}")
+            return [original_query]
+    
+    def _identify_query_components(self, query: str) -> dict:
+        """Identify different components in a complex query for better breakdown."""
+        components = {
+            'processes': [],
+            'documents': [],
+            'contacts': [],
+            'eligibility': [],
+            'timelines': [],
+            'benefits': []
+        }
+        
+        # Define keywords for different component types
+        process_keywords = ['process', 'procedure', 'steps', 'how to', 'submit', 'apply', 'claim', 'update', 'change', 'enroll']
+        document_keywords = ['documents', 'forms', 'papers', 'certificate', 'proof', 'evidence', 'requirements']
+        contact_keywords = ['email', 'phone', 'contact', 'grievance', 'customer service', 'support', 'helpline']
+        eligibility_keywords = ['eligibility', 'criteria', 'qualify', 'eligible', 'conditions', 'requirements']
+        timeline_keywords = ['timeline', 'period', 'duration', 'time', 'days', 'months', 'waiting', 'grace']
+        benefit_keywords = ['benefits', 'coverage', 'limits', 'amount', 'reimbursement', 'claim amount']
+        
+        query_lower = query.lower()
+        
+        # Check for process-related content
+        if any(keyword in query_lower for keyword in process_keywords):
+            components['processes'].append('process identification')
+        
+        # Check for document-related content
+        if any(keyword in query_lower for keyword in document_keywords):
+            components['documents'].append('document requirements')
+        
+        # Check for contact-related content
+        if any(keyword in query_lower for keyword in contact_keywords):
+            components['contacts'].append('contact information')
+        
+        # Check for eligibility-related content
+        if any(keyword in query_lower for keyword in eligibility_keywords):
+            components['eligibility'].append('eligibility criteria')
+        
+        # Check for timeline-related content
+        if any(keyword in query_lower for keyword in timeline_keywords):
+            components['timelines'].append('timeline information')
+        
+        # Check for benefit-related content
+        if any(keyword in query_lower for keyword in benefit_keywords):
+            components['benefits'].append('benefit details')
+        
+        return components

RAG/rag_modules/reranking_manager.py

@@ -0,0 +1,63 @@
+"""
+Reranking Module for Advanced RAG
+Handles result reranking using cross-encoder models.
+"""
+
+from typing import List, Dict
+from sentence_transformers import CrossEncoder
+from config.config import ENABLE_RERANKING, RERANKER_MODEL, RERANK_TOP_K
+
+
+class RerankingManager:
+    """Manages result reranking using cross-encoder models."""
+    
+    def __init__(self):
+        """Initialize the reranking manager."""
+        self.reranker_model = None
+        if ENABLE_RERANKING:
+            self._init_reranker_model()
+        print("✅ Reranking Manager initialized")
+    
+    def _init_reranker_model(self):
+        """Initialize the reranker model."""
+        print(f"🔄 Loading reranker model: {RERANKER_MODEL}")
+        self.reranker_model = CrossEncoder(RERANKER_MODEL)
+        print(f"✅ Reranker model loaded successfully")
+    
+    async def rerank_results(self, query: str, search_results: List[Dict]) -> List[Dict]:
+        """Rerank search results using cross-encoder."""
+        if not ENABLE_RERANKING or not self.reranker_model or len(search_results) <= 1:
+            return search_results
+        
+        try:
+            # Prepare pairs for reranking
+            query_doc_pairs = []
+            for result in search_results:
+                doc_text = result['payload'].get('text', '')[:512]  # Limit text length
+                query_doc_pairs.append([query, doc_text])
+            
+            # Get reranking scores
+            rerank_scores = self.reranker_model.predict(query_doc_pairs)
+            
+            # Combine with original scores
+            for i, result in enumerate(search_results):
+                original_score = result.get('score', 0)
+                rerank_score = float(rerank_scores[i])
+                
+                # Weighted combination of original and rerank scores
+                result['rerank_score'] = rerank_score
+                result['final_score'] = 0.3 * original_score + 0.7 * rerank_score
+            
+            # Sort by final score
+            reranked_results = sorted(
+                search_results,
+                key=lambda x: x['final_score'],
+                reverse=True
+            )
+            
+            print(f"🎯 Reranked {len(search_results)} results")
+            return reranked_results[:RERANK_TOP_K]
+            
+        except Exception as e:
+            print(f"⚠️ Reranking failed: {e}")
+            return search_results[:RERANK_TOP_K]

RAG/rag_modules/search_manager.py

@@ -0,0 +1,334 @@
+"""
+Search Module for Advanced RAG
+Handles hybrid search combining BM25 and semantic search with score fusion.
+"""
+
+import re
+import time
+import numpy as np
+from typing import List, Dict, Any
+from pathlib import Path
+from rank_bm25 import BM25Okapi
+from qdrant_client import QdrantClient
+
+from config.config import (
+    OUTPUT_DIR, TOP_K, SCORE_THRESHOLD, ENABLE_HYBRID_SEARCH,
+    BM25_WEIGHT, SEMANTIC_WEIGHT, USE_TOTAL_BUDGET_APPROACH
+)
+
+
+class SearchManager:
+    """Manages hybrid search operations combining BM25 and semantic search."""
+    
+    def __init__(self, embedding_manager):
+        """Initialize the search manager."""
+        self.embedding_manager = embedding_manager
+        self.base_db_path = Path(OUTPUT_DIR)
+        self.qdrant_clients = {}
+        self.bm25_indexes = {}  # Cache BM25 indexes per document
+        self.document_chunks = {}  # Cache chunks for BM25
+        print("✅ Search Manager initialized")
+    
+    def get_qdrant_client(self, doc_id: str) -> QdrantClient:
+        """Get or create Qdrant client for a specific document."""
+        if doc_id not in self.qdrant_clients:
+            db_path = self.base_db_path / f"{doc_id}_collection.db"
+            if not db_path.exists():
+                raise FileNotFoundError(f"Database not found for document {doc_id}")
+            self.qdrant_clients[doc_id] = QdrantClient(path=str(db_path))
+        return self.qdrant_clients[doc_id]
+    
+    def _load_bm25_index(self, doc_id: str):
+        """Load or create BM25 index for a document."""
+        if doc_id not in self.bm25_indexes:
+            print(f"🔄 Loading BM25 index for {doc_id}")
+            
+            # Get all chunks from Qdrant
+            client = self.get_qdrant_client(doc_id)
+            collection_name = f"{doc_id}_collection"
+            
+            try:
+                # Get all points from collection
+                result = client.scroll(
+                    collection_name=collection_name,
+                    limit=10000,  # Adjust based on your chunk count
+                    with_payload=True,
+                    with_vectors=False
+                )
+                
+                chunks = []
+                chunk_ids = []
+                
+                for point in result[0]:
+                    chunk_text = point.payload.get('text', '')
+                    chunks.append(chunk_text)
+                    chunk_ids.append(point.id)
+                
+                # Tokenize chunks for BM25
+                tokenized_chunks = [self._tokenize_text(chunk) for chunk in chunks]
+                
+                # Create BM25 index
+                self.bm25_indexes[doc_id] = BM25Okapi(tokenized_chunks)
+                self.document_chunks[doc_id] = {
+                    'chunks': chunks,
+                    'chunk_ids': chunk_ids,
+                    'tokenized_chunks': tokenized_chunks
+                }
+                
+                print(f"✅ BM25 index loaded for {doc_id} with {len(chunks)} chunks")
+                
+            except Exception as e:
+                print(f"❌ Error loading BM25 index for {doc_id}: {e}")
+                # Fallback: empty index
+                self.bm25_indexes[doc_id] = BM25Okapi([[]])
+                self.document_chunks[doc_id] = {'chunks': [], 'chunk_ids': [], 'tokenized_chunks': []}
+    
+    def _tokenize_text(self, text: str) -> List[str]:
+        """Simple tokenization for BM25."""
+        # Remove special characters and convert to lowercase
+        text = re.sub(r'[^\w\s]', ' ', text.lower())
+        # Split and filter empty tokens
+        tokens = [token for token in text.split() if len(token) > 2]
+        return tokens
+    
+    async def hybrid_search(self, queries: List[str], doc_id: str, top_k: int = TOP_K) -> List[Dict]:
+        """
+        Perform hybrid search combining BM25 and semantic search.
+        Optimized for focused sub-queries from query breakdown.
+        Uses total budget approach to distribute retrieval across queries.
+        """
+        collection_name = f"{doc_id}_collection"
+        client = self.get_qdrant_client(doc_id)
+        
+        # Ensure BM25 index is loaded
+        if doc_id not in self.bm25_indexes:
+            self._load_bm25_index(doc_id)
+        
+        # Calculate per-query budget based on approach
+        if USE_TOTAL_BUDGET_APPROACH and len(queries) > 1:
+            per_query_budget = max(1, top_k // len(queries))
+            extra_budget = top_k % len(queries)  # Distribute remaining budget
+            print(f"🎯 Total Budget Approach: Distributing {top_k} candidates across {len(queries)} queries")
+            print(f"   📊 Base budget per query: {per_query_budget}")
+            if extra_budget > 0:
+                print(f"   ➕ Extra budget for first {extra_budget} queries: +1 each")
+        else:
+            per_query_budget = top_k
+            extra_budget = 0
+            print(f"🔍 Per-Query Approach: Each query gets {per_query_budget} candidates")
+        
+        all_candidates = {}  # point_id -> {'score': float, 'payload': dict, 'source': str}
+        query_performance = {}  # Track performance of each sub-query
+        
+        print(f"🔍 Running hybrid search with {len(queries)} focused queries...")
+        
+        for query_idx, query in enumerate(queries):
+            query_candidates = 0
+            query_start = time.time()
+            
+            # Calculate this query's budget
+            if USE_TOTAL_BUDGET_APPROACH and len(queries) > 1:
+                query_budget = per_query_budget + (1 if query_idx < extra_budget else 0)
+                search_limit = query_budget * 2  # Get extra for better selection
+            else:
+                query_budget = per_query_budget
+                search_limit = query_budget * 2
+            
+            print(f"   Q{query_idx+1} Budget: {query_budget} candidates (searching {search_limit})")
+            
+            # 1. Semantic Search
+            if ENABLE_HYBRID_SEARCH or not ENABLE_HYBRID_SEARCH:  # Always do semantic
+                try:
+                    query_vector = await self.embedding_manager.encode_query(query)
+                    semantic_results = client.search(
+                        collection_name=collection_name,
+                        query_vector=query_vector,
+                        limit=search_limit,  # Use query-specific limit
+                        score_threshold=SCORE_THRESHOLD
+                    )
+                    
+                    # Process semantic results with budget limit
+                    semantic_count = 0
+                    for result in semantic_results:
+                        if USE_TOTAL_BUDGET_APPROACH and semantic_count >= query_budget:
+                            break  # Respect budget limit
+                            
+                        point_id = str(result.id)
+                        semantic_score = float(result.score)
+                        
+                        if point_id not in all_candidates:
+                            all_candidates[point_id] = {
+                                'semantic_score': 0,
+                                'bm25_score': 0,
+                                'payload': result.payload,
+                                'fusion_score': 0,
+                                'contributing_queries': []
+                            }
+                        
+                        # Use max score across queries for semantic, but track which queries contributed
+                        if semantic_score > all_candidates[point_id]['semantic_score']:
+                            all_candidates[point_id]['semantic_score'] = semantic_score
+                        
+                        all_candidates[point_id]['contributing_queries'].append({
+                            'query_idx': query_idx,
+                            'query_text': query[:50] + '...' if len(query) > 50 else query,
+                            'semantic_score': semantic_score,
+                            'type': 'semantic'
+                        })
+                        query_candidates += 1
+                        semantic_count += 1
+                
+                except Exception as e:
+                    print(f"⚠️ Semantic search failed for query '{query[:50]}...': {e}")
+            
+            # 2. BM25 Search (if enabled)
+            if ENABLE_HYBRID_SEARCH and doc_id in self.bm25_indexes:
+                try:
+                    tokenized_query = self._tokenize_text(query)
+                    bm25_scores = self.bm25_indexes[doc_id].get_scores(tokenized_query)
+                    
+                    # Get top BM25 results with budget consideration
+                    chunk_data = self.document_chunks[doc_id]
+                    bm25_top_indices = np.argsort(bm25_scores)[::-1][:search_limit]
+                    
+                    # Process BM25 results with budget limit
+                    bm25_count = 0
+                    for idx in bm25_top_indices:
+                        if USE_TOTAL_BUDGET_APPROACH and bm25_count >= query_budget:
+                            break  # Respect budget limit
+                            
+                        if idx < len(chunk_data['chunk_ids']) and bm25_scores[idx] > 0:
+                            point_id = str(chunk_data['chunk_ids'][idx])
+                            bm25_score = float(bm25_scores[idx])
+                            
+                            if point_id not in all_candidates:
+                                all_candidates[point_id] = {
+                                    'semantic_score': 0,
+                                    'bm25_score': 0,
+                                    'payload': {'text': chunk_data['chunks'][idx]},
+                                    'fusion_score': 0,
+                                    'contributing_queries': []
+                                }
+                            
+                            # Use max score across queries for BM25, but track which queries contributed
+                            if bm25_score > all_candidates[point_id]['bm25_score']:
+                                all_candidates[point_id]['bm25_score'] = bm25_score
+                            
+                            all_candidates[point_id]['contributing_queries'].append({
+                                'query_idx': query_idx,
+                                'query_text': query[:50] + '...' if len(query) > 50 else query,
+                                'bm25_score': bm25_score,
+                                'type': 'bm25'
+                            })
+                            query_candidates += 1
+                            bm25_count += 1
+                
+                except Exception as e:
+                    print(f"⚠️ BM25 search failed for query '{query[:50]}...': {e}")
+            
+            # Track query performance with budget info
+            query_time = time.time() - query_start
+            query_performance[query_idx] = {
+                'query': query[:80] + '...' if len(query) > 80 else query,
+                'candidates_found': query_candidates,
+                'budget_allocated': query_budget if USE_TOTAL_BUDGET_APPROACH else 'unlimited',
+                'time': query_time
+            }
+        
+        # 3. Score Fusion (Reciprocal Rank Fusion + Weighted Combination)
+        self._apply_score_fusion(all_candidates)
+        
+        # 4. Sort by fusion score and return top results
+        sorted_candidates = sorted(
+            all_candidates.items(),
+            key=lambda x: x[1]['fusion_score'],
+            reverse=True
+        )
+        
+        # Convert to result format with enhanced metadata
+        hybrid_results = []
+        for point_id, data in sorted_candidates[:top_k]:
+            hybrid_results.append({
+                'id': point_id,
+                'score': data['fusion_score'],
+                'payload': data['payload'],
+                'semantic_score': data['semantic_score'],
+                'bm25_score': data['bm25_score'],
+                'contributing_queries': data['contributing_queries']
+            })
+        
+        # Log performance summary
+        approach_name = "Total Budget" if USE_TOTAL_BUDGET_APPROACH else "Per-Query"
+        print(f"🔍 Hybrid search completed ({approach_name} Approach):")
+        print(f"   📊 {len(all_candidates)} total candidates from {len(queries)} focused queries")
+        print(f"   🎯 Top {len(hybrid_results)} results selected")
+        
+        # Log per-query performance with budget info
+        total_budget_used = 0
+        for idx, perf in query_performance.items():
+            budget_info = f" (budget: {perf['budget_allocated']})" if USE_TOTAL_BUDGET_APPROACH else ""
+            print(f"   Q{idx+1}: {perf['candidates_found']} candidates{budget_info} in {perf['time']:.3f}s")
+            print(f"        Query: {perf['query']}")
+            if USE_TOTAL_BUDGET_APPROACH and isinstance(perf['budget_allocated'], int):
+                total_budget_used += perf['candidates_found']
+        
+        if USE_TOTAL_BUDGET_APPROACH:
+            print(f"   💰 Total budget efficiency: {total_budget_used}/{top_k} candidates used")
+        
+        return hybrid_results
+    
+    def _apply_score_fusion(self, candidates: Dict):
+        """Apply advanced score fusion techniques."""
+        if not candidates:
+            return
+        
+        # Normalize scores
+        semantic_scores = [data['semantic_score'] for data in candidates.values() if data['semantic_score'] > 0]
+        bm25_scores = [data['bm25_score'] for data in candidates.values() if data['bm25_score'] > 0]
+        
+        # Min-Max normalization
+        if semantic_scores:
+            sem_min, sem_max = min(semantic_scores), max(semantic_scores)
+            sem_range = sem_max - sem_min if sem_max > sem_min else 1
+        else:
+            sem_min, sem_range = 0, 1
+            
+        if bm25_scores:
+            bm25_min, bm25_max = min(bm25_scores), max(bm25_scores)
+            bm25_range = bm25_max - bm25_min if bm25_max > bm25_min else 1
+        else:
+            bm25_min, bm25_range = 0, 1
+        
+        # Calculate fusion scores
+        for point_id, data in candidates.items():
+            # Normalize scores
+            norm_semantic = (data['semantic_score'] - sem_min) / sem_range if data['semantic_score'] > 0 else 0
+            norm_bm25 = (data['bm25_score'] - bm25_min) / bm25_range if data['bm25_score'] > 0 else 0
+            
+            # Weighted combination
+            if ENABLE_HYBRID_SEARCH:
+                fusion_score = (SEMANTIC_WEIGHT * norm_semantic) + (BM25_WEIGHT * norm_bm25)
+            else:
+                fusion_score = norm_semantic
+            
+            # Add reciprocal rank fusion bonus (helps with ranking diversity)
+            rank_bonus = 1.0 / (1.0 + max(norm_semantic, norm_bm25) * 10)
+            fusion_score += rank_bonus * 0.1
+            
+            data['fusion_score'] = fusion_score
+    
+    def cleanup(self):
+        """Cleanup search manager resources."""
+        print("🧹 Cleaning up Search Manager resources...")
+        
+        # Close all Qdrant clients
+        for client in self.qdrant_clients.values():
+            try:
+                client.close()
+            except Exception:
+                pass
+        
+        self.qdrant_clients.clear()
+        self.bm25_indexes.clear()
+        self.document_chunks.clear()
+        print("✅ Search Manager cleanup completed")

api/__init__.py

@@ -0,0 +1 @@
+# API Package

api/api.py

@@ -0,0 +1,498 @@
+from fastapi import FastAPI, HTTPException, Depends, Query
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from pydantic import BaseModel, HttpUrl
+from typing import List, Dict, Any, Optional
+import tempfile
+import os
+import hashlib
+import asyncio
+import aiohttp
+import time
+from contextlib import asynccontextmanager
+
+from RAG.advanced_rag_processor import AdvancedRAGProcessor
+from preprocessing.preprocessing import DocumentPreprocessor
+from logger.logger import rag_logger
+from LLM.llm_handler import llm_handler
+from LLM.tabular_answer import get_answer_for_tabluar
+from LLM.image_answerer import get_answer_for_image
+from LLM.one_shotter import get_oneshot_answer
+from config.config import *
+import config.config as config
+
+# Initialize security
+security = HTTPBearer()
+admin_security = HTTPBearer()
+
+def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    """Verify the bearer token for main API."""
+    if credentials.credentials != BEARER_TOKEN:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid authentication token"
+        )
+    return credentials.credentials
+
+def verify_admin_token(credentials: HTTPAuthorizationCredentials = Depends(admin_security)):
+    """Verify the bearer token for admin endpoints."""
+    if credentials.credentials != "9420689497":
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid admin authentication token"
+        )
+    return credentials.credentials
+
+# Pydantic models for request/response
+class ProcessDocumentRequest(BaseModel):
+    documents: HttpUrl  # URL to the PDF document
+    questions: List[str]  # List of questions to answer
+
+class ProcessDocumentResponse(BaseModel):
+    answers: List[str]
+
+class HealthResponse(BaseModel):
+    status: str
+    message: str
+
+class PreprocessingResponse(BaseModel):
+    status: str
+    message: str
+    doc_id: str
+    chunk_count: int
+
+class LogsResponse(BaseModel):
+    export_timestamp: str
+    metadata: Dict[str, Any]
+    logs: List[Dict[str, Any]]
+
+class LogsSummaryResponse(BaseModel):
+    summary: Dict[str, Any]
+
+# Global instances
+rag_processor: Optional[AdvancedRAGProcessor] = None
+document_preprocessor: Optional[DocumentPreprocessor] = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Initialize and cleanup the RAG processor."""
+    global rag_processor, document_preprocessor
+    
+    # Startup
+    print("🚀 Initializing Advanced RAG System...")
+    rag_processor = AdvancedRAGProcessor()  # Use advanced processor for better accuracy
+    document_preprocessor = DocumentPreprocessor()
+    print("✅ Advanced RAG System initialized successfully")
+    
+    yield
+    
+    # Shutdown
+    print("🔄 Shutting down RAG System...")
+    if rag_processor:
+        rag_processor.cleanup()
+    print("✅ Cleanup completed")
+
+# FastAPI app with lifespan management
+app = FastAPI(
+    title="Advanced RAG API",
+    description="API for document processing and question answering using RAG",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+@app.get("/health", response_model=HealthResponse)
+async def health_check():
+    """Health check endpoint."""
+    return HealthResponse(
+        status="healthy",
+        message="RAG API is running successfully"
+    )
+
+@app.post("/hackrx/run", response_model=ProcessDocumentResponse)
+async def process_document(
+    request: ProcessDocumentRequest, 
+    token: str = Depends(verify_token)
+):
+    """
+    Process a PDF document and answer questions about it.
+    
+    This endpoint implements an optimized flow:
+    1. Check if the document is already processed (pre-computed embeddings)
+    2. If yes, use existing embeddings for fast retrieval + generation
+    3. If no, run full RAG pipeline (download + process + embed + store + answer)
+    
+    Args:
+        request: Contains document URL and list of questions
+        token: Bearer token for authentication
+        
+    Returns:
+        ProcessDocumentResponse: List of answers corresponding to the questions
+    """
+    global rag_processor, document_preprocessor
+    
+    if not rag_processor or not document_preprocessor:
+        raise HTTPException(
+            status_code=503, 
+            detail="RAG system not initialized"
+        )
+    
+    # Start timing and logging
+    start_time = time.time()
+    document_url = str(request.documents)
+    questions = request.questions
+    # Initialize answers for safe logging in finally
+    final_answers = []
+    status = "success"
+    error_message = None
+    doc_id = None
+    was_preprocessed = False
+    
+    # Initialize enhanced logging
+    request_id = rag_logger.generate_request_id()
+    rag_logger.start_request_timing(request_id)
+    
+    try:
+        print(f"📋 [{request_id}] Processing document: {document_url[:50]}...")
+        print(f"🤔 [{request_id}] Number of questions: {len(questions)}")
+        print(f"")
+        print(f"🚀 [{request_id}] ===== STARTING RAG PIPELINE =====")
+        print(f"📌 [{request_id}] PRIORITY 1: Checking stored embeddings database...")
+        
+        # Generate document ID
+        doc_id = document_preprocessor.generate_doc_id(document_url)
+        
+        # Step 1: Check if document is already processed (stored embeddings)
+        is_processed = document_preprocessor.is_document_processed(document_url)
+        was_preprocessed = is_processed
+        
+        if is_processed:
+            print(f"✅ [{request_id}] ✅ FOUND STORED EMBEDDINGS for {doc_id}")
+            print(f"⚡ [{request_id}] Using fast path with pre-computed embeddings")
+            # Fast path: Use existing embeddings
+            doc_info = document_preprocessor.get_document_info(document_url)
+            print(f"📊 [{request_id}] Using existing collection with {doc_info.get('chunk_count', 'N/A')} chunks")
+        else:
+            print(f"❌ [{request_id}] No stored embeddings found for {doc_id}")
+            print(f"📌 [{request_id}] PRIORITY 2: Running full RAG pipeline (download + process + embed)...")
+            # Full path: Download and process document
+            resp = await document_preprocessor.process_document(document_url)
+            
+            # Handle different return formats: [content, type] or [content, type, no_cleanup_flag]
+            if isinstance(resp, list):
+                content, _type = resp[0], resp[1]
+                if content == 'unsupported':
+                    # Unsupported file type: respond gracefully without throwing server error
+                    msg = f"Unsupported file type: {_type.lstrip('.')}"
+                    final_answers = [msg]
+                    status = "success"  # ensure no 500 is raised in the finally block
+                    return ProcessDocumentResponse(answers=final_answers)
+                
+                if _type == "image":
+                    try:
+                        final_answers = get_answer_for_image(content, questions)
+                        status = "success"
+                        return ProcessDocumentResponse(answers=final_answers)
+                    finally:
+                        # Clean up the image file after processing
+                        if os.path.exists(content):
+                            os.unlink(content)
+                            print(f"🗑️ Cleaned up image file: {content}")
+
+                if _type == "tabular":
+                    final_answers = get_answer_for_tabluar(content, questions)
+                    status = "success"
+                    return ProcessDocumentResponse(answers=final_answers)
+                
+                if _type == "oneshot":
+                    # Process questions in batches for oneshot
+                    tasks = [
+                        get_oneshot_answer(content, questions[i:i + 3])
+                        for i in range(0, len(questions), 3)
+                    ]
+                    
+                    # Run all batches in parallel
+                    results = await asyncio.gather(*tasks)
+                    
+                    # Flatten results
+                    final_answers = [ans for batch in results for ans in batch]
+                    status = "success"
+                    return ProcessDocumentResponse(answers=final_answers)          
+            else:
+                doc_id = resp
+
+            print(f"✅ [{request_id}] Document {doc_id} processed and stored")
+        
+        # Answer all questions using parallel processing for better latency
+        print(f"🚀 [{request_id}] Processing {len(questions)} questions in parallel...")
+        
+        async def answer_single_question(question: str, index: int) -> tuple[str, Dict[str, float]]:
+            """Answer a single question with error handling and timing."""
+            try:
+                question_start = time.time()
+                print(f"❓ [{request_id}] Q{index+1}: {question[:50]}...")
+                
+                answer, pipeline_timings = await rag_processor.answer_question(
+                    question=question,
+                    doc_id=doc_id,
+                    logger=rag_logger,
+                    request_id=request_id
+                )
+                
+                question_time = time.time() - question_start
+                
+                # Log question timing
+                rag_logger.log_question_timing(
+                    request_id, index, question, answer, question_time, pipeline_timings
+                )
+                
+                print(f"✅ [{request_id}] Q{index+1} completed in {question_time:.4f}s")
+                return answer, pipeline_timings
+            except Exception as e:
+                print(f"❌ [{request_id}] Q{index+1} Error: {str(e)}")
+                return f"I encountered an error while processing this question: {str(e)}", {}
+        
+        
+        # Process questions in parallel with controlled concurrency
+        semaphore = asyncio.Semaphore(3)  # Reduced concurrency for better logging visibility
+        
+        async def bounded_answer(question: str, index: int) -> tuple[str, Dict[str, float]]:
+            async with semaphore:
+                return await answer_single_question(question, index)
+        
+        # Execute all questions concurrently
+        tasks = [
+            bounded_answer(question, i) 
+            for i, question in enumerate(questions)
+        ]
+        
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        
+        # Handle any exceptions in answers
+        final_answers = []
+        error_count = 0
+        for i, result in enumerate(results):
+            if isinstance(result, Exception):
+                error_count += 1
+                final_answers.append(f"Error processing question {i+1}: {str(result)}")
+            else:
+                answer, _ = result
+                final_answers.append(answer)
+        
+        # Determine final status
+        if error_count == 0:
+            status = "success"
+        elif error_count == len(questions):
+            status = "error"
+        else:
+            status = "partial"
+        
+        print(f"✅ [{request_id}] Successfully processed {len(questions) - error_count}/{len(questions)} questions")
+        
+    except Exception as e:
+        print(f"❌ [{request_id}] Error processing request: {str(e)}")
+        status = "error"
+        error_message = str(e)
+        final_answers = [f"Error: {str(e)}" for _ in questions]
+    
+    finally:
+        # End request timing and get detailed timing data
+        timing_data = rag_logger.end_request_timing(request_id)
+        
+        # Log the request with enhanced timing
+        processing_time = time.time() - start_time
+        logged_request_id = rag_logger.log_request(
+            document_url=document_url,
+            questions=questions,
+            answers=final_answers,
+            processing_time=processing_time,
+            status=status,
+            error_message=error_message,
+            document_id=doc_id,
+            was_preprocessed=was_preprocessed,
+            timing_data=timing_data
+        )
+        
+        print(f"📊 Request logged with ID: {logged_request_id} (Status: {status}, Time: {processing_time:.2f}s)")
+        
+        if status == "error":
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to process document: {error_message}"
+            )
+    
+    return ProcessDocumentResponse(answers=final_answers)
+
+@app.post("/preprocess", response_model=PreprocessingResponse)
+async def preprocess_document(document_url: str, force: bool = False, token: str = Depends(verify_admin_token)):
+    """
+    Preprocess a document (for batch preprocessing).
+    
+    Args:
+        document_url: URL of the PDF to preprocess
+        force: Whether to reprocess if already processed
+        
+    Returns:
+        PreprocessingResponse: Status and document info
+    """
+    global document_preprocessor
+    
+    if not document_preprocessor:
+        raise HTTPException(
+            status_code=503,
+            detail="Document preprocessor not initialized"
+        )
+    
+    try:
+        doc_id = await document_preprocessor.process_document(document_url, force)
+        doc_info = document_preprocessor.get_document_info(document_url)
+        
+        return PreprocessingResponse(
+            status="success",
+            message=f"Document processed successfully",
+            doc_id=doc_id,
+            chunk_count=doc_info.get("chunk_count", 0)
+        )
+        
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to preprocess document: {str(e)}"
+        )
+
+@app.get("/collections")
+async def list_collections(token: str = Depends(verify_admin_token)):
+    """List all available document collections."""
+    global document_preprocessor
+    
+    if not document_preprocessor:
+        raise HTTPException(
+            status_code=503,
+            detail="Document preprocessor not initialized"
+        )
+    
+    try:
+        processed_docs = document_preprocessor.list_processed_documents()
+        return {"collections": processed_docs}
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to list collections: {str(e)}"
+        )
+
+@app.get("/collections/stats")
+async def get_collection_stats(token: str = Depends(verify_admin_token)):
+    """Get statistics about all collections."""
+    global document_preprocessor
+    
+    if not document_preprocessor:
+        raise HTTPException(
+            status_code=503,
+            detail="Document preprocessor not initialized"
+        )
+    
+    try:
+        stats = document_preprocessor.get_collection_stats()
+        return stats
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to get collection stats: {str(e)}"
+        )
+    
+# Logging Endpoints
+@app.get("/logs", response_model=LogsResponse)
+async def get_logs(
+    token: str = Depends(verify_admin_token),
+    limit: Optional[int] = Query(None, description="Maximum number of logs to return"),
+    minutes: Optional[int] = Query(None, description="Get logs from last N minutes"),
+    document_url: Optional[str] = Query(None, description="Filter logs by document URL")
+):
+    """
+    Export all API request logs as JSON.
+    
+    Query Parameters:
+        limit: Maximum number of recent logs to return
+        minutes: Get logs from the last N minutes
+        document_url: Filter logs for a specific document URL
+    
+    Returns:
+        LogsResponse: Complete logs export with metadata
+    """
+    try:
+        if document_url:
+            # Get logs for specific document
+            logs = rag_logger.get_logs_by_document(document_url)
+            metadata = {
+                "filtered_by": "document_url",
+                "document_url": document_url,
+                "total_logs": len(logs)
+            }
+            return LogsResponse(
+                export_timestamp=rag_logger.export_logs()["export_timestamp"],
+                metadata=metadata,
+                logs=logs
+            )
+        
+        elif minutes:
+            # Get recent logs
+            logs = rag_logger.get_recent_logs(minutes)
+            metadata = {
+                "filtered_by": "time_range",
+                "minutes": minutes,
+                "total_logs": len(logs)
+            }
+            return LogsResponse(
+                export_timestamp=rag_logger.export_logs()["export_timestamp"],
+                metadata=metadata,
+                logs=logs
+            )
+        
+        else:
+            # Get all logs (with optional limit)
+            if limit:
+                logs = rag_logger.get_logs(limit)
+                metadata = rag_logger.get_logs_summary()
+                metadata["limited_to"] = limit
+            else:
+                logs_export = rag_logger.export_logs()
+                return LogsResponse(**logs_export)
+            
+            return LogsResponse(
+                export_timestamp=rag_logger.export_logs()["export_timestamp"],
+                metadata=metadata,
+                logs=logs
+            )
+    
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to export logs: {str(e)}"
+        )
+
+@app.get("/logs/summary", response_model=LogsSummaryResponse)
+async def get_logs_summary(token: str = Depends(verify_admin_token)):
+    """
+    Get summary statistics of all logs.
+    
+    Returns:
+        LogsSummaryResponse: Summary statistics
+    """
+    try:
+        summary = rag_logger.get_logs_summary()
+        return LogsSummaryResponse(summary=summary)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to get logs summary: {str(e)}"
+        )
+
+if __name__ == "__main__":
+    import uvicorn
+    
+    # Run the FastAPI server
+    uvicorn.run(
+        "api:app",
+        host=API_HOST,
+        port=API_PORT,
+        reload=API_RELOAD,
+        log_level="info"
+    )

config/__init__.py

@@ -0,0 +1 @@
+# Config Package

config/config.py

@@ -0,0 +1,142 @@
+# RAG Configuration File
+# Update these settings as needed
+
+import os
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Common LLM Settings
+MAX_TOKENS = 1200 
+TEMPERATURE = 0.4
+
+# OCR Settings
+OCR_SPACE_API_KEY = os.getenv("OCR_SPACE_API_KEY", "")
+
+# OpenAI Settings
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") 
+OPENAI_MODEL = "gpt-3.5-turbo"
+
+# Gemini Settings
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "") 
+GEMINI_MODEL = "gemini-1.5-flash"
+
+# Groq Settings
+GROQ_API_KEY = os.getenv("GROQ_API_KEY") 
+GROQ_MODEL = os.getenv("GROQ_MODEL", "llama3-70b-8192")
+
+GROQ_API_KEY_LITE = os.getenv("GROQ_API_KEY_LITE") 
+GROQ_MODEL_LITE = "llama3-8b-8192"
+
+# API Authentication
+BEARER_TOKEN = os.getenv("BEARER_TOKEN", "c6cee5b5046310e401632a7effe9c684d071a9ef5ce09b96c9ec5c3ebd13085e")
+
+# Chunking (TOKEN-BASED)
+CHUNK_SIZE = 1600  #
+CHUNK_OVERLAP = 200  
+
+# Retrieval Settings
+TOP_K = 12
+SCORE_THRESHOLD = 0.3 
+RERANK_TOP_K = 9  
+BM25_WEIGHT = 0.3 
+SEMANTIC_WEIGHT = 0.7
+
+# Advanced RAG Settings
+ENABLE_RERANKING = True
+ENABLE_HYBRID_SEARCH = True
+ENABLE_QUERY_EXPANSION = True
+RERANKER_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+QUERY_EXPANSION_COUNT = 3 
+MAX_CONTEXT_LENGTH = 15000
+
+USE_TOTAL_BUDGET_APPROACH = True  
+
+# Embedding Settings
+EMBEDDING_MODEL = "BAAI/bge-large-en"  
+BATCH_SIZE = 16  
+
+# Paths
+OUTPUT_DIR = os.getenv("RAG_EMBEDDINGS_PATH", "./RAG/rag_embeddings")
+
+# API Settings
+API_HOST = "0.0.0.0"
+API_PORT = 8000
+API_RELOAD = True
+
+# Multi-LLM failover system
+sequence = ["primary", "secondary", "ternary", "quaternary", "quinary", "senary", "septenary", "octonary", "nonary", "denary"]
+
+def get_provider_configs():
+    """
+    Get configurations for all provider instances with failover support.
+    Supports multiple instances of each provider type for reliability.
+    """
+    configs = {
+        "groq": [],
+        "gemini": [],
+        "openai": []
+    }
+    
+    # Groq configurations with multiple API keys for failover
+    DEFAULT_GROQ_MODEL = "qwen/qwen3-32b"
+    configs["groq"] = [{
+        "name": sequence[i-1],
+        "api_key": os.getenv(f"GROQ_API_KEY_{i}"),
+        "model": os.getenv(f"GROQ_MODEL_{i}", DEFAULT_GROQ_MODEL)} for i in range(1, 10) if os.getenv(f"GROQ_API_KEY_{i}", "")
+    ]
+    
+    # Add main GROQ key as primary
+    if os.getenv("GROQ_API_KEY"):
+        configs["groq"].insert(0, {
+            "name": "main",
+            "api_key": os.getenv("GROQ_API_KEY"),
+            "model": DEFAULT_GROQ_MODEL
+        })
+    
+    # Gemini configurations with multiple API keys for failover
+    DEFAULT_GEMINI_MODEL = "gemini-1.5-flash"
+    configs["gemini"] = [{
+            "name": sequence[i-1],
+            "api_key": os.getenv(f"GEMINI_API_KEY_{i}"),
+            "model": os.getenv(f"GEMINI_MODEL_{i}", DEFAULT_GEMINI_MODEL)
+        } for i in range(1, 10) if os.getenv(f"GEMINI_API_KEY_{i}", "")
+    ]
+    
+    # Add main GEMINI key as primary
+    if os.getenv("GEMINI_API_KEY"):
+        configs["gemini"].insert(0, {
+            "name": "main",
+            "api_key": os.getenv("GEMINI_API_KEY"),
+            "model": DEFAULT_GEMINI_MODEL
+        })
+    
+    # OpenAI configurations with multiple API keys for failover
+    DEFAULT_OPENAI_MODEL = "gpt-4o-mini"
+    configs["openai"] = [{
+            "name": sequence[i-1],
+            "api_key": os.getenv(f"OPENAI_API_KEY_{i}"),
+            "model": os.getenv(f"OPENAI_MODEL_{i}", DEFAULT_OPENAI_MODEL)
+        } for i in range(1, 10) if os.getenv(f"OPENAI_API_KEY_{i}", "")
+    ]
+    
+    # Add main OPENAI key as primary
+    if os.getenv("OPENAI_API_KEY"):
+        configs["openai"].insert(0, {
+            "name": "main",
+            "api_key": os.getenv("OPENAI_API_KEY"),
+            "model": DEFAULT_OPENAI_MODEL
+        })
+    
+    return configs
+
+# Specialized API keys for different tasks
+GROQ_API_KEY_TABULAR = os.getenv("GROQ_API_KEY_TABULAR", GROQ_API_KEY)
+GEMINI_API_KEY_IMAGE = os.getenv("GEMINI_API_KEY_IMAGE", GEMINI_API_KEY)
+GEMINI_API_KEY_MULTILINGUAL = os.getenv("GEMINI_API_KEY_MULTILINGUAL", GEMINI_API_KEY)
+
+# Validation (optional - comment out for production)
+# assert OPENAI_API_KEY, "OPENAI KEY NOT SET"
+# assert GEMINI_API_KEY, "GEMINI KEY NOT SET" 
+# assert GROQ_API_KEY, "GROQ KEY NOT SET"

logger/__init__.py

@@ -0,0 +1 @@
+# Logger Package

logger/logger.py

@@ -0,0 +1,340 @@
+"""
+Enhanced in-memory logging system for RAG API with detailed pipeline timing.
+Since HuggingFace doesn't allow persistent file storage, logs are stored in memory.
+"""
+
+import json
+import time
+from datetime import datetime
+from typing import List, Dict, Any, Optional
+from dataclasses import dataclass, asdict, field
+import threading
+
+@dataclass
+class PipelineTimings:
+    """Detailed timing for each stage of the RAG pipeline."""
+    query_expansion_time: float = 0.0
+    hybrid_search_time: float = 0.0
+    semantic_search_time: float = 0.0
+    bm25_search_time: float = 0.0
+    score_fusion_time: float = 0.0
+    reranking_time: float = 0.0
+    context_creation_time: float = 0.0
+    llm_generation_time: float = 0.0
+    total_pipeline_time: float = 0.0
+
+@dataclass
+class LogEntry:
+    """Enhanced structure for a single log entry with detailed timing."""
+    timestamp: str
+    request_id: str
+    document_url: str
+    questions: List[str]
+    answers: List[str]
+    processing_time_seconds: float
+    total_questions: int
+    status: str  # 'success', 'error', 'partial'
+    error_message: Optional[str] = None
+    document_id: Optional[str] = None
+    was_preprocessed: bool = False
+    # Enhanced timing details
+    request_start_time: str = ""
+    request_end_time: str = ""
+    pipeline_timings: Dict[str, Any] = field(default_factory=dict)
+    # Per-question timings
+    question_timings: List[Dict[str, Any]] = field(default_factory=list)
+
+class RAGLogger:
+    """Enhanced in-memory logging system for RAG API requests with detailed pipeline timing."""
+    
+    def __init__(self):
+        self.logs: List[LogEntry] = []
+        self.server_start_time = datetime.now().isoformat()
+        self.request_counter = 0
+        self._lock = threading.Lock()
+        # Active request tracking for timing
+        self._active_requests: Dict[str, Dict[str, Any]] = {}
+    
+    def generate_request_id(self) -> str:
+        """Generate a unique request ID."""
+        with self._lock:
+            self.request_counter += 1
+            return f"req_{self.request_counter:06d}"
+    
+    def start_request_timing(self, request_id: str) -> None:
+        """Start timing for a new request."""
+        self._active_requests[request_id] = {
+            'start_time': time.time(),
+            'start_timestamp': datetime.now().isoformat(),
+            'pipeline_stages': {},
+            'question_timings': []
+        }
+    
+    def log_pipeline_stage(self, request_id: str, stage_name: str, duration: float) -> None:
+        """Log the timing for a specific pipeline stage."""
+        if request_id in self._active_requests:
+            self._active_requests[request_id]['pipeline_stages'][stage_name] = {
+                'duration_seconds': round(duration, 4),
+                'timestamp': datetime.now().isoformat()
+            }
+            print(f"⏱️  [{request_id}] {stage_name}: {duration:.4f}s")
+    
+    def log_question_timing(self, request_id: str, question_index: int, question: str, 
+                           answer: str, duration: float, pipeline_timings: Dict[str, float]) -> None:
+        """Log timing for individual question processing."""
+        if request_id in self._active_requests:
+            question_timing = {
+                'question_index': question_index,
+                'question': question[:100] + "..." if len(question) > 100 else question,
+                'answer_length': len(answer),
+                'total_time_seconds': round(duration, 4),
+                'pipeline_breakdown': {k: round(v, 4) for k, v in pipeline_timings.items()},
+                'timestamp': datetime.now().isoformat()
+            }
+            self._active_requests[request_id]['question_timings'].append(question_timing)
+            
+            # Enhanced console logging
+            print(f"\n❓ [{request_id}] Question {question_index + 1}: {question[:60]}...")
+            print(f"   📊 Processing time: {duration:.4f}s")
+            if pipeline_timings:
+                breakdown_str = " | ".join([f"{k}: {v:.4f}s" for k, v in pipeline_timings.items() if v > 0])
+                if breakdown_str:
+                    print(f"   ⚙️  Pipeline breakdown: {breakdown_str}")
+            print(f"   💬 Answer length: {len(answer)} characters")
+    
+    def end_request_timing(self, request_id: str) -> Dict[str, Any]:
+        """End timing for a request and return timing data."""
+        if request_id not in self._active_requests:
+            return {}
+        
+        request_data = self._active_requests[request_id]
+        total_time = time.time() - request_data['start_time']
+        
+        timing_data = {
+            'start_time': request_data['start_timestamp'],
+            'end_time': datetime.now().isoformat(),
+            'total_time_seconds': round(total_time, 4),
+            'pipeline_stages': request_data['pipeline_stages'],
+            'question_timings': request_data['question_timings']
+        }
+        
+        # Cleanup
+        del self._active_requests[request_id]
+        
+        return timing_data
+    
+    def log_request(
+        self, 
+        document_url: str, 
+        questions: List[str], 
+        answers: List[str], 
+        processing_time: float,
+        status: str = "success",
+        error_message: Optional[str] = None,
+        document_id: Optional[str] = None,
+        was_preprocessed: bool = False,
+        timing_data: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Log a RAG API request with enhanced timing information.
+        
+        Args:
+            document_url: URL of the document processed
+            questions: List of questions asked
+            answers: List of answers generated
+            processing_time: Time taken in seconds
+            status: Request status ('success', 'error', 'partial')
+            error_message: Error message if any
+            document_id: Generated document ID
+            was_preprocessed: Whether document was already processed
+            timing_data: Detailed timing breakdown from pipeline
+        
+        Returns:
+            str: Request ID
+        """
+        request_id = self.generate_request_id()
+        
+        # Extract timing information
+        pipeline_timings = {}
+        question_timings = []
+        request_start_time = ""
+        request_end_time = ""
+        
+        if timing_data:
+            request_start_time = timing_data.get('start_time', '')
+            request_end_time = timing_data.get('end_time', '')
+            pipeline_timings = timing_data.get('pipeline_stages', {})
+            question_timings = timing_data.get('question_timings', [])
+        
+        log_entry = LogEntry(
+            timestamp=datetime.now().isoformat(),
+            request_id=request_id,
+            document_url=document_url,
+            questions=questions,
+            answers=answers,
+            processing_time_seconds=round(processing_time, 2),
+            total_questions=len(questions),
+            status=status,
+            error_message=error_message,
+            document_id=document_id,
+            was_preprocessed=was_preprocessed,
+            request_start_time=request_start_time,
+            request_end_time=request_end_time,
+            pipeline_timings=pipeline_timings,
+            question_timings=question_timings
+        )
+        
+        with self._lock:
+            self.logs.append(log_entry)
+        
+        # Enhanced console logging summary
+        print(f"\n📊 [{request_id}] REQUEST COMPLETED:")
+        print(f"   🕐 Duration: {processing_time:.2f}s")
+        print(f"   📄 Document: {document_url[:60]}...")
+        print(f"   ❓ Questions processed: {len(questions)}")
+        print(f"   ✅ Status: {status.upper()}")
+        
+        if pipeline_timings:
+            print(f"   ⚙️  Pipeline performance:")
+            for stage, data in pipeline_timings.items():
+                duration = data.get('duration_seconds', 0)
+                print(f"      • {stage.replace('_', ' ').title()}: {duration:.4f}s")
+        
+        if error_message:
+            print(f"   ❌ Error: {error_message}")
+        
+        print(f"   🆔 Request ID: {request_id}")
+        print("   " + "="*50)
+        
+        return request_id
+    
+    def get_logs(self, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Get all logs as a list of dictionaries.
+        
+        Args:
+            limit: Maximum number of logs to return (most recent first)
+        
+        Returns:
+            List of log entries as dictionaries
+        """
+        with self._lock:
+            logs_list = [asdict(log) for log in self.logs]
+        
+        # Return most recent first
+        logs_list.reverse()
+        
+        if limit:
+            logs_list = logs_list[:limit]
+        
+        return logs_list
+    
+    def get_logs_summary(self) -> Dict[str, Any]:
+        """Get summary statistics of all logs."""
+        with self._lock:
+            total_requests = len(self.logs)
+            if total_requests == 0:
+                return {
+                    "server_start_time": self.server_start_time,
+                    "total_requests": 0,
+                    "successful_requests": 0,
+                    "error_requests": 0,
+                    "average_processing_time": 0,
+                    "total_questions_processed": 0,
+                    "total_documents_processed": 0
+                }
+            
+            successful_requests = len([log for log in self.logs if log.status == "success"])
+            error_requests = len([log for log in self.logs if log.status == "error"])
+            total_processing_time = sum(log.processing_time_seconds for log in self.logs)
+            total_questions = sum(log.total_questions for log in self.logs)
+            unique_documents = len(set(log.document_url for log in self.logs))
+            preprocessed_count = len([log for log in self.logs if log.was_preprocessed])
+            
+            # Enhanced timing statistics
+            pipeline_times = []
+            question_times = []
+            stage_times = {'query_expansion': [], 'hybrid_search': [], 'reranking': [], 
+                          'context_creation': [], 'llm_generation': []}
+            
+            for log in self.logs:
+                # Collect question timing data
+                for q_timing in log.question_timings:
+                    question_times.append(q_timing.get('total_time_seconds', 0))
+                    # Collect stage-specific timings
+                    breakdown = q_timing.get('pipeline_breakdown', {})
+                    for stage, duration in breakdown.items():
+                        if stage in stage_times:
+                            stage_times[stage].append(duration)
+            
+            # Calculate averages for each stage
+            avg_stage_times = {}
+            for stage, times in stage_times.items():
+                if times:
+                    avg_stage_times[f'avg_{stage}_time'] = round(sum(times) / len(times), 4)
+                    avg_stage_times[f'max_{stage}_time'] = round(max(times), 4)
+                else:
+                    avg_stage_times[f'avg_{stage}_time'] = 0
+                    avg_stage_times[f'max_{stage}_time'] = 0
+            
+            return {
+                "server_start_time": self.server_start_time,
+                "total_requests": total_requests,
+                "successful_requests": successful_requests,
+                "error_requests": error_requests,
+                "partial_requests": total_requests - successful_requests - error_requests,
+                "success_rate": round((successful_requests / total_requests) * 100, 2),
+                "average_processing_time": round(total_processing_time / total_requests, 2),
+                "total_questions_processed": total_questions,
+                "total_documents_processed": unique_documents,
+                "documents_already_preprocessed": preprocessed_count,
+                "documents_newly_processed": total_requests - preprocessed_count,
+                "average_question_time": round(sum(question_times) / len(question_times), 4) if question_times else 0,
+                "pipeline_performance": avg_stage_times
+            }
+    
+    def export_logs(self) -> Dict[str, Any]:
+        """
+        Export all logs in a structured format for external consumption.
+        
+        Returns:
+            Dict containing metadata and all logs
+        """
+        summary = self.get_logs_summary()
+        logs = self.get_logs()
+        
+        return {
+            "export_timestamp": datetime.now().isoformat(),
+            "metadata": summary,
+            "logs": logs
+        }
+    
+    def get_logs_by_document(self, document_url: str) -> List[Dict[str, Any]]:
+        """Get all logs for a specific document URL."""
+        with self._lock:
+            filtered_logs = [
+                asdict(log) for log in self.logs 
+                if log.document_url == document_url
+            ]
+        
+        # Return most recent first
+        filtered_logs.reverse()
+        return filtered_logs
+    
+    def get_recent_logs(self, minutes: int = 60) -> List[Dict[str, Any]]:
+        """Get logs from the last N minutes."""
+        cutoff_time = datetime.now().timestamp() - (minutes * 60)
+        
+        with self._lock:
+            recent_logs = []
+            for log in self.logs:
+                log_time = datetime.fromisoformat(log.timestamp).timestamp()
+                if log_time >= cutoff_time:
+                    recent_logs.append(asdict(log))
+        
+        # Return most recent first
+        recent_logs.reverse()
+        return recent_logs
+
+# Global logger instance
+rag_logger = RAGLogger()

preprocessing/__init__.py

@@ -0,0 +1,23 @@
+# Preprocessing package
+
+from .preprocessing import DocumentPreprocessor
+from .preprocessing_modules import (
+    PDFDownloader,
+    TextExtractor,
+    TextChunker,
+    EmbeddingManager,
+    VectorStorage,
+    MetadataManager,
+    ModularDocumentPreprocessor
+)
+
+__all__ = [
+    'DocumentPreprocessor',
+    'PDFDownloader',
+    'TextExtractor',
+    'TextChunker', 
+    'EmbeddingManager',
+    'VectorStorage',
+    'MetadataManager',
+    'ModularDocumentPreprocessor'
+]

preprocessing/preprocessing.py

@@ -0,0 +1,63 @@
+import os
+import asyncio
+from typing import List, Dict, Any
+
+from config.config import *
+from .preprocessing_modules.modular_preprocessor import ModularDocumentPreprocessor
+
+# For backward compatibility, create an alias
+class DocumentPreprocessor(ModularDocumentPreprocessor):
+    """Backward compatibility alias for the modular document preprocessor."""
+    pass
+
+# CLI interface for preprocessing
+async def main():
+    """Main function for command-line usage."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Document Preprocessing for RAG")
+    parser.add_argument("--url", type=str, help="Single PDF URL to process")
+    parser.add_argument("--urls-file", type=str, help="File containing PDF URLs (one per line)")
+    parser.add_argument("--force", action="store_true", help="Force reprocessing even if already processed")
+    parser.add_argument("--list", action="store_true", help="List all processed documents")
+    parser.add_argument("--stats", action="store_true", help="Show collection statistics")
+    
+    args = parser.parse_args()
+    
+    preprocessor = DocumentPreprocessor()
+    
+    if args.list:
+        docs = preprocessor.list_processed_documents()
+        print(f"\n📚 Processed Documents ({len(docs)}):")
+        for doc_id, info in docs.items():
+            print(f"  • {doc_id}: {info['document_url'][:50]}... ({info.get('chunk_count', 'N/A')} chunks)")
+    
+    elif args.stats:
+        stats = preprocessor.get_collection_stats()
+        print(f"\n📊 Collection Statistics:")
+        print(f"  • Total documents: {stats['total_documents']}")
+        print(f"  • Total collections: {stats['total_collections']}")
+        print(f"  • Total chunks: {stats['total_chunks']}")
+    
+    elif args.url:
+        await preprocessor.process_document(args.url, args.force)
+    
+    elif args.urls_file:
+        if not os.path.exists(args.urls_file):
+            print(f"❌ File not found: {args.urls_file}")
+            return
+        
+        with open(args.urls_file, 'r') as f:
+            urls = [line.strip() for line in f if line.strip()]
+        
+        if urls:
+            await preprocessor.process_multiple_documents(urls, args.force)
+        else:
+            print("❌ No URLs found in file")
+    
+    else:
+        print("❌ Please provide --url, --urls-file, --list, or --stats")
+        parser.print_help()
+
+if __name__ == "__main__":
+    asyncio.run(main())

preprocessing/preprocessing_modules/__init__.py

@@ -0,0 +1,29 @@
+# Preprocessing modules
+
+from .pdf_downloader import PDFDownloader
+from .file_downloader import FileDownloader
+from .text_extractor import TextExtractor
+from .text_chunker import TextChunker
+from .embedding_manager import EmbeddingManager
+from .vector_storage import VectorStorage
+from .metadata_manager import MetadataManager
+from .modular_preprocessor import ModularDocumentPreprocessor
+from .docx_extractor import extract_docx
+from .pptx_extractor import extract_pptx
+from .xlsx_extractor import extract_xlsx
+from .image_extractor import extract_image_content
+
+__all__ = [
+    'PDFDownloader',
+    'FileDownloader',
+    'TextExtractor', 
+    'TextChunker',
+    'EmbeddingManager',
+    'VectorStorage',
+    'MetadataManager',
+    'ModularDocumentPreprocessor',
+    'extract_docx',
+    'extract_pptx', 
+    'extract_xlsx',
+    'extract_image_content'
+]

preprocessing/preprocessing_modules/docx_extractor.py

@@ -0,0 +1,94 @@
+from docx import Document
+from docx.document import Document as _Document
+from docx.table import Table
+from docx.text.paragraph import Paragraph
+from typing import Union, List, Dict, Any
+from PIL import Image
+from io import BytesIO
+import pytesseract
+import os
+
+from zipfile import ZipFile
+from lxml import etree
+from pathlib import Path
+import io
+
+def extract_docx(docx_input) -> str:
+    """Extract text from DOCX files with table and text handling."""
+    zipf = ZipFile(docx_input)
+    xml_content = zipf.read("word/document.xml")
+    tree = etree.fromstring(xml_content)
+
+    ns = {
+        "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+        "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
+        "wps": "http://schemas.microsoft.com/office/word/2010/wordprocessingShape"
+    }
+
+    text_blocks = []
+
+    # Extract all tables with gridSpan handling
+    tables = tree.xpath("//w:tbl", namespaces=ns)
+    table_elements = set(tables)
+    table_index = 0
+    
+    for tbl in tables:
+        rows = tbl.xpath("./w:tr", namespaces=ns)
+        sub_tables = []
+        current_table = []
+
+        prev_col_count = None
+        for row in rows:
+            row_texts = []
+            cells = row.xpath("./w:tc", namespaces=ns)
+            col_count = 0
+            
+            for cell in cells:
+                cell_text = ""
+                paragraphs = cell.xpath(".//w:p", namespaces=ns)
+                for para in paragraphs:
+                    text_nodes = para.xpath(".//w:t", namespaces=ns)
+                    para_text = "".join(node.text for node in text_nodes if node.text)
+                    if para_text.strip():
+                        cell_text += para_text + " "
+                
+                # Handle gridSpan (merged cells)
+                gridspan_elem = cell.xpath(".//w:gridSpan", namespaces=ns)
+                span = int(gridspan_elem[0].get(ns["w"] + "val", "1")) if gridspan_elem else 1
+                
+                row_texts.append(cell_text.strip())
+                col_count += span
+
+            if row_texts and any(text.strip() for text in row_texts):
+                if prev_col_count is not None and col_count != prev_col_count:
+                    # Column count changed, save current table and start new one
+                    if current_table:
+                        sub_tables.append(current_table)
+                        current_table = []
+                
+                current_table.append(row_texts)
+                prev_col_count = col_count
+
+        if current_table:
+            sub_tables.append(current_table)
+
+        # Format tables
+        for sub_table in sub_tables:
+            table_text = f"\\n--- Table {table_index + 1} ---\\n"
+            for row in sub_table:
+                table_text += " | ".join(row) + "\\n"
+            text_blocks.append(table_text)
+            table_index += 1
+
+    # Extract non-table paragraphs
+    paragraphs = tree.xpath("//w:p", namespaces=ns)
+    for para in paragraphs:
+        # Check if paragraph is inside a table
+        is_in_table = any(table in para.xpath("ancestor::*") for table in table_elements)
+        if not is_in_table:
+            text_nodes = para.xpath(".//w:t", namespaces=ns)
+            para_text = "".join(node.text for node in text_nodes if node.text)
+            if para_text.strip():
+                text_blocks.append(para_text.strip())
+
+    return "\\n\\n".join(text_blocks)

preprocessing/preprocessing_modules/embedding_manager.py

@@ -0,0 +1,118 @@
+"""
+Embedding Manager Module
+
+Handles creation of embeddings for text chunks using sentence transformers.
+"""
+
+import asyncio
+import numpy as np
+from typing import List
+from sentence_transformers import SentenceTransformer
+from config.config import EMBEDDING_MODEL, BATCH_SIZE
+
+
+class EmbeddingManager:
+    """Handles embedding creation for text chunks."""
+    
+    def __init__(self):
+        """Initialize the embedding manager."""
+        self.embedding_model = None
+        self._init_embedding_model()
+    
+    def _init_embedding_model(self):
+        """Initialize the embedding model."""
+        print(f"🔄 Loading embedding model: {EMBEDDING_MODEL}")
+        self.embedding_model = SentenceTransformer(EMBEDDING_MODEL)
+        print(f"✅ Embedding model loaded successfully")
+    
+    async def create_embeddings(self, chunks: List[str]) -> np.ndarray:
+        """
+        Create embeddings for text chunks.
+        
+        Args:
+            chunks: List of text chunks to embed
+            
+        Returns:
+            np.ndarray: Array of embeddings with shape (num_chunks, embedding_dim)
+        """
+        print(f"🧠 Creating embeddings for {len(chunks)} chunks")
+        
+        if not chunks:
+            raise ValueError("No chunks provided for embedding creation")
+        
+        def create_embeddings_sync():
+            """Synchronous embedding creation to run in thread pool."""
+            embeddings = self.embedding_model.encode(
+                chunks,
+                batch_size=BATCH_SIZE,
+                show_progress_bar=True,
+                normalize_embeddings=True
+            )
+            return np.array(embeddings).astype("float32")
+        
+        # Run in thread pool to avoid blocking the event loop
+        loop = asyncio.get_event_loop()
+        embeddings = await loop.run_in_executor(None, create_embeddings_sync)
+        
+        print(f"✅ Created embeddings with shape: {embeddings.shape}")
+        return embeddings
+    
+    def get_embedding_dimension(self) -> int:
+        """
+        Get the dimension of embeddings produced by the model.
+        
+        Returns:
+            int: Embedding dimension
+        """
+        if self.embedding_model is None:
+            raise RuntimeError("Embedding model not initialized")
+        
+        # Get dimension from model
+        return self.embedding_model.get_sentence_embedding_dimension()
+    
+    def validate_embeddings(self, embeddings: np.ndarray, expected_count: int) -> bool:
+        """
+        Validate that embeddings have the expected shape and properties.
+        
+        Args:
+            embeddings: The embeddings array to validate
+            expected_count: Expected number of embeddings
+            
+        Returns:
+            bool: True if embeddings are valid, False otherwise
+        """
+        if embeddings is None:
+            return False
+        
+        if embeddings.shape[0] != expected_count:
+            print(f"❌ Embedding count mismatch: expected {expected_count}, got {embeddings.shape[0]}")
+            return False
+        
+        if embeddings.dtype != np.float32:
+            print(f"❌ Embedding dtype mismatch: expected float32, got {embeddings.dtype}")
+            return False
+        
+        # Check for NaN or infinite values
+        if np.any(np.isnan(embeddings)) or np.any(np.isinf(embeddings)):
+            print("❌ Embeddings contain NaN or infinite values")
+            return False
+        
+        print(f"✅ Embeddings validation passed: {embeddings.shape}")
+        return True
+    
+    def get_model_info(self) -> dict:
+        """
+        Get information about the embedding model.
+        
+        Returns:
+            dict: Model information
+        """
+        if self.embedding_model is None:
+            return {"model_name": EMBEDDING_MODEL, "status": "not_loaded"}
+        
+        return {
+            "model_name": EMBEDDING_MODEL,
+            "embedding_dimension": self.get_embedding_dimension(),
+            "max_sequence_length": getattr(self.embedding_model, 'max_seq_length', 'unknown'),
+            "status": "loaded"
+        }

preprocessing/preprocessing_modules/file_downloader.py

@@ -0,0 +1,108 @@
+import aiohttp
+import asyncio
+import tempfile
+import os
+import re
+from urllib.parse import urlparse
+from typing import List, Tuple
+
+class FileDownloader:
+    """Enhanced file downloader that supports multiple file types."""
+
+    async def download_file(self, url: str, timeout: int = 300, max_retries: int = 3) -> Tuple[str, str]:
+        """Download any file type from a URL to a temporary file with enhanced error handling."""
+        print(f"📥 Downloading file from: {url[:60]}...")
+
+        for attempt in range(max_retries):
+            try:
+                timeout_config = aiohttp.ClientTimeout(
+                    total=timeout,
+                    connect=30,
+                    sock_read=120
+                )
+
+                async with aiohttp.ClientSession(timeout=timeout_config) as session:
+                    print(f"   Attempt {attempt + 1}/{max_retries} (timeout: {timeout}s)")
+
+                    async with session.get(url) as response:
+                        if response.status != 200:
+                            raise Exception(f"Failed to download file: HTTP {response.status}")
+
+                        # Extract filename from header or URL
+                        cd = response.headers.get('Content-Disposition', '')
+                        filename_match = re.findall('filename="?([^"]+)"?', cd)
+                        if filename_match:
+                            filename = filename_match[0]
+                        else:
+                            from urllib.parse import unquote
+                            path = urlparse(url).path
+                            filename = os.path.basename(unquote(path))  # Decode URL encoding
+
+                        if not filename:
+                            filename = "downloaded_file"
+
+                        ext = os.path.splitext(filename)[1]
+                        if not ext:
+                            return url, "url"
+
+                        print(f"   📁 Detected filename: {filename}, extension: {ext}")
+
+                        # Check if file type is supported
+                        supported_extensions = ['.pdf', '.docx', '.pptx', '.png', '.xlsx', '.jpeg', '.jpg', '.txt', '.csv']
+                        if ext not in supported_extensions:
+                            # Return extension without dot for consistency
+                            ext_without_dot = ext[1:] if ext.startswith('.') else ext
+                            print(f"   ❌ File type not supported: {ext}")
+                            return 'not supported', ext_without_dot
+
+                        # Get content length
+                        content_length = response.headers.get('content-length')
+                        if content_length:
+                            total_size = int(content_length)
+                            print(f"   File size: {total_size / (1024 * 1024):.1f} MB")
+
+                        # Create temp file with same extension
+                        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=ext, prefix="download_")
+
+                        # Write to file
+                        downloaded = 0
+                        async for chunk in response.content.iter_chunked(16384):
+                            temp_file.write(chunk)
+                            downloaded += len(chunk)
+
+                            if content_length and downloaded % (1024 * 1024) == 0:
+                                progress = (downloaded / total_size) * 100
+                                print(f"   Progress: {progress:.1f}% ({downloaded / (1024*1024):.1f} MB)")
+
+                        temp_file.close()
+                        print(f"✅ File downloaded successfully: {temp_file.name}")
+                        # Return extension without the dot for consistency with modular_preprocessor
+                        ext_without_dot = ext[1:] if ext.startswith('.') else ext
+                        return temp_file.name, ext_without_dot
+
+            except asyncio.TimeoutError:
+                print(f"   ⏰ Timeout on attempt {attempt + 1}")
+                if attempt < max_retries - 1:
+                    wait_time = (attempt + 1) * 30
+                    print(f"   ⏳ Waiting {wait_time}s before retry...")
+                    await asyncio.sleep(wait_time)
+                continue
+
+            except Exception as e:
+                print(f"   ❌ Error on attempt {attempt + 1}: {str(e)}")
+                if attempt < max_retries - 1:
+                    wait_time = (attempt + 1) * 15
+                    print(f"   ⏳ Waiting {wait_time}s before retry...")
+                    await asyncio.sleep(wait_time)
+                continue
+
+        raise Exception(f"Failed to download file after {max_retries} attempts")
+
+    def cleanup_temp_file(self, temp_path: str) -> None:
+        """Clean up temporary file."""
+        try:
+            if os.path.exists(temp_path):
+                os.unlink(temp_path)
+                print(f"🗑️ Cleaned up temporary file: {temp_path}")
+        except Exception as e:
+            print(f"⚠️ Warning: Could not cleanup temp file {temp_path}: {e}")

preprocessing/preprocessing_modules/image_extractor.py

@@ -0,0 +1,120 @@
+import cv2
+import pytesseract
+import numpy as np
+import pandas as pd
+from PIL import Image, ImageFile
+from typing import List, Dict, Any
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+def load_local_image(path: str) -> np.ndarray:
+    """Load image from local path."""
+    img = Image.open(path).convert("RGB")
+    return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
+
+def sort_contours(cnts, method="top-to-bottom"):
+    """Sort contours based on the specified method."""
+    reverse = False
+    i = 1 if method == "top-to-bottom" or method == "bottom-to-top" else 0
+    if method == "right-to-left" or method == "bottom-to-top":
+        reverse = True
+    boundingBoxes = [cv2.boundingRect(c) for c in cnts]
+    (cnts, boundingBoxes) = zip(*sorted(zip(cnts, boundingBoxes),
+                                        key=lambda b: b[1][i], reverse=reverse))
+    return cnts, boundingBoxes
+
+def extract_cells_from_grid(table_img: np.ndarray) -> pd.DataFrame:
+    """Extract table structure from image using OpenCV."""
+    gray = cv2.cvtColor(table_img, cv2.COLOR_BGR2GRAY)
+    _, binary = cv2.threshold(~gray, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
+
+    # Detect horizontal lines
+    horizontal = binary.copy()
+    cols = horizontal.shape[1]
+    horizontal_size = cols // 15
+    horizontal_structure = cv2.getStructuringElement(cv2.MORPH_RECT, (horizontal_size, 1))
+    horizontal = cv2.erode(horizontal, horizontal_structure)
+    horizontal = cv2.dilate(horizontal, horizontal_structure)
+
+    # Detect vertical lines
+    vertical = binary.copy()
+    rows = vertical.shape[0]
+    vertical_size = rows // 15
+    vertical_structure = cv2.getStructuringElement(cv2.MORPH_RECT, (1, vertical_size))
+    vertical = cv2.erode(vertical, vertical_structure)
+    vertical = cv2.dilate(vertical, vertical_structure)
+
+    # Combine mask
+    mask = cv2.add(horizontal, vertical)
+    contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+
+    cells = []
+    for contour in contours:
+        x, y, w, h = cv2.boundingRect(contour)
+        if w > 30 and h > 20:  # Filter small contours
+            cell_img = table_img[y:y+h, x:x+w]
+            try:
+                text = pytesseract.image_to_string(cell_img, config='--psm 7').strip()
+                cells.append({'x': x, 'y': y, 'w': w, 'h': h, 'text': text})
+            except:
+                cells.append({'x': x, 'y': y, 'w': w, 'h': h, 'text': ''})
+
+    # Sort cells by position to create table structure
+    cells.sort(key=lambda cell: (cell['y'], cell['x']))
+    
+    # Group cells into rows
+    rows = []
+    current_row = []
+    current_y = 0
+    
+    for cell in cells:
+        if abs(cell['y'] - current_y) > 20:  # New row threshold
+            if current_row:
+                rows.append(current_row)
+            current_row = [cell]
+            current_y = cell['y']
+        else:
+            current_row.append(cell)
+    
+    if current_row:
+        rows.append(current_row)
+
+    # Convert to DataFrame
+    table_data = []
+    for row in rows:
+        row_data = [cell['text'] for cell in sorted(row, key=lambda c: c['x'])]
+        table_data.append(row_data)
+
+    if table_data:
+        max_cols = max(len(row) for row in table_data)
+        for row in table_data:
+            while len(row) < max_cols:
+                row.append('')
+        return pd.DataFrame(table_data)
+    else:
+        return pd.DataFrame()
+
+def extract_image_content(image_path: str) -> str:
+    """Extract text content from images using OCR."""
+    try:
+        # Load image
+        img = load_local_image(image_path)
+        
+        # Basic OCR
+        text = pytesseract.image_to_string(img)
+        
+        # Try to detect if it's a table
+        if '|' in text or '\\t' in text or len(text.split('\\n')) > 3:
+            # Try table extraction
+            try:
+                table_df = extract_cells_from_grid(img)
+                if not table_df.empty:
+                    table_text = "\\n".join([" | ".join(row) for row in table_df.values])
+                    return f"[Table detected]\\n{table_text}\\n\\n[OCR Text]\\n{text}"
+            except:
+                pass
+        
+        return text.strip() if text.strip() else "[No text detected in image]"
+        
+    except Exception as e:
+        return f"[Error processing image: {str(e)}]"

preprocessing/preprocessing_modules/metadata_manager.py

@@ -0,0 +1,262 @@
+"""
+Metadata Manager Module
+
+Handles document metadata storage and retrieval operations.
+"""
+
+import json
+import asyncio
+import hashlib
+from typing import List, Dict, Any
+from pathlib import Path
+from config.config import EMBEDDING_MODEL, CHUNK_SIZE, CHUNK_OVERLAP
+
+
+class MetadataManager:
+    """Handles document metadata operations."""
+    
+    def __init__(self, base_db_path: Path):
+        """
+        Initialize the metadata manager.
+        
+        Args:
+            base_db_path: Base path for storing metadata files
+        """
+        self.base_db_path = base_db_path
+        self.processed_docs_file = self.base_db_path / "processed_documents.json"
+        self.processed_docs = self._load_processed_docs()
+    
+    def _load_processed_docs(self) -> Dict[str, Dict]:
+        """Load the registry of processed documents."""
+        if self.processed_docs_file.exists():
+            try:
+                with open(self.processed_docs_file, 'r', encoding='utf-8') as f:
+                    return json.load(f)
+            except Exception as e:
+                print(f"⚠️ Warning: Could not load processed docs registry: {e}")
+        return {}
+    
+    def _save_processed_docs(self):
+        """Save the registry of processed documents."""
+        try:
+            with open(self.processed_docs_file, 'w', encoding='utf-8') as f:
+                json.dump(self.processed_docs, f, indent=2, ensure_ascii=False)
+        except Exception as e:
+            print(f"⚠️ Warning: Could not save processed docs registry: {e}")
+    
+    def generate_doc_id(self, document_url: str) -> str:
+        """
+        Generate a unique document ID from the URL.
+        
+        Args:
+            document_url: URL of the document
+            
+        Returns:
+            str: Unique document ID
+        """
+        url_hash = hashlib.md5(document_url.encode()).hexdigest()[:12]
+        return f"doc_{url_hash}"
+    
+    def is_document_processed(self, document_url: str) -> bool:
+        """
+        Check if a document has already been processed.
+        
+        Args:
+            document_url: URL of the document
+            
+        Returns:
+            bool: True if document is already processed
+        """
+        doc_id = self.generate_doc_id(document_url)
+        return doc_id in self.processed_docs
+    
+    def get_document_info(self, document_url: str) -> Dict[str, Any]:
+        """
+        Get information about a processed document.
+        
+        Args:
+            document_url: URL of the document
+            
+        Returns:
+            Dict[str, Any]: Document information or empty dict if not found
+        """
+        doc_id = self.generate_doc_id(document_url)
+        return self.processed_docs.get(doc_id, {})
+    
+    def save_document_metadata(self, chunks: List[str], doc_id: str, document_url: str):
+        """
+        Save document metadata to JSON file and update registry.
+        
+        Args:
+            chunks: List of text chunks
+            doc_id: Document identifier
+            document_url: Original document URL
+        """
+        # Calculate statistics
+        total_chars = sum(len(chunk) for chunk in chunks)
+        total_words = sum(len(chunk.split()) for chunk in chunks)
+        avg_chunk_size = total_chars / len(chunks) if chunks else 0
+        
+        # Create metadata object
+        metadata = {
+            "doc_id": doc_id,
+            "document_url": document_url,
+            "chunk_count": len(chunks),
+            "total_chars": total_chars,
+            "total_words": total_words,
+            "avg_chunk_size": avg_chunk_size,
+            "processed_at": asyncio.get_event_loop().time(),
+            "embedding_model": EMBEDDING_MODEL,
+            "chunk_size": CHUNK_SIZE,
+            "chunk_overlap": CHUNK_OVERLAP,
+            "processing_config": {
+                "chunk_size": CHUNK_SIZE,
+                "chunk_overlap": CHUNK_OVERLAP,
+                "embedding_model": EMBEDDING_MODEL
+            }
+        }
+        
+        # Save individual document metadata
+        metadata_path = self.base_db_path / f"{doc_id}_metadata.json"
+        try:
+            with open(metadata_path, "w", encoding="utf-8") as f:
+                json.dump(metadata, f, indent=2, ensure_ascii=False)
+            print(f"✅ Saved individual metadata for {doc_id}")
+        except Exception as e:
+            print(f"⚠️ Warning: Could not save individual metadata for {doc_id}: {e}")
+        
+        # Update processed documents registry
+        self.processed_docs[doc_id] = {
+            "document_url": document_url,
+            "chunk_count": len(chunks),
+            "processed_at": metadata["processed_at"],
+            "collection_name": f"{doc_id}_collection",
+            "total_chars": total_chars,
+            "total_words": total_words
+        }
+        self._save_processed_docs()
+        
+        print(f"✅ Updated registry for document {doc_id}")
+    
+    def get_document_metadata(self, doc_id: str) -> Dict[str, Any]:
+        """
+        Load individual document metadata from file.
+        
+        Args:
+            doc_id: Document identifier
+            
+        Returns:
+            Dict[str, Any]: Document metadata or empty dict if not found
+        """
+        metadata_path = self.base_db_path / f"{doc_id}_metadata.json"
+        
+        if not metadata_path.exists():
+            return {}
+        
+        try:
+            with open(metadata_path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        except Exception as e:
+            print(f"⚠️ Warning: Could not load metadata for {doc_id}: {e}")
+            return {}
+    
+    def list_processed_documents(self) -> Dict[str, Dict]:
+        """
+        List all processed documents.
+        
+        Returns:
+            Dict[str, Dict]: Copy of processed documents registry
+        """
+        return self.processed_docs.copy()
+    
+    def get_collection_stats(self) -> Dict[str, Any]:
+        """
+        Get statistics about all collections.
+        
+        Returns:
+            Dict[str, Any]: Collection statistics
+        """
+        stats = {
+            "total_documents": len(self.processed_docs),
+            "total_collections": 0,
+            "total_chunks": 0,
+            "total_characters": 0,
+            "total_words": 0,
+            "documents": []
+        }
+        
+        for doc_id, info in self.processed_docs.items():
+            collection_path = self.base_db_path / f"{info['collection_name']}.db"
+            if collection_path.exists():
+                stats["total_collections"] += 1
+                stats["total_chunks"] += info.get("chunk_count", 0)
+                stats["total_characters"] += info.get("total_chars", 0)
+                stats["total_words"] += info.get("total_words", 0)
+                
+                stats["documents"].append({
+                    "doc_id": doc_id,
+                    "url": info["document_url"],
+                    "chunk_count": info.get("chunk_count", 0),
+                    "total_chars": info.get("total_chars", 0),
+                    "total_words": info.get("total_words", 0),
+                    "processed_at": info.get("processed_at", "unknown")
+                })
+        
+        # Add averages
+        if stats["total_documents"] > 0:
+            stats["avg_chunks_per_doc"] = stats["total_chunks"] / stats["total_documents"]
+            stats["avg_chars_per_doc"] = stats["total_characters"] / stats["total_documents"]
+            stats["avg_words_per_doc"] = stats["total_words"] / stats["total_documents"]
+        
+        return stats
+    
+    def remove_document_metadata(self, doc_id: str) -> bool:
+        """
+        Remove document metadata and registry entry.
+        
+        Args:
+            doc_id: Document identifier
+            
+        Returns:
+            bool: True if successfully removed, False otherwise
+        """
+        try:
+            # Remove individual metadata file
+            metadata_path = self.base_db_path / f"{doc_id}_metadata.json"
+            if metadata_path.exists():
+                metadata_path.unlink()
+                print(f"🗑️ Removed metadata file for {doc_id}")
+            
+            # Remove from registry
+            if doc_id in self.processed_docs:
+                del self.processed_docs[doc_id]
+                self._save_processed_docs()
+                print(f"🗑️ Removed registry entry for {doc_id}")
+            
+            return True
+            
+        except Exception as e:
+            print(f"❌ Error removing metadata for {doc_id}: {e}")
+            return False
+    
+    def update_document_status(self, doc_id: str, status_info: Dict[str, Any]):
+        """
+        Update status information for a document.
+        
+        Args:
+            doc_id: Document identifier
+            status_info: Status information to update
+        """
+        if doc_id in self.processed_docs:
+            self.processed_docs[doc_id].update(status_info)
+            self._save_processed_docs()
+            print(f"✅ Updated status for document {doc_id}")
+    
+    def get_registry_path(self) -> str:
+        """
+        Get the path to the processed documents registry.
+        
+        Returns:
+            str: Path to registry file
+        """
+        return str(self.processed_docs_file)

preprocessing/preprocessing_modules/modular_preprocessor.py

@@ -0,0 +1,290 @@
+"""
+Modular Document Preprocessor
+
+Main orchestrator class that uses all preprocessing modules to process documents.
+"""
+
+import os
+import asyncio
+from typing import List, Dict, Any, Union
+from pathlib import Path
+
+from config.config import OUTPUT_DIR
+from .pdf_downloader import PDFDownloader
+from .file_downloader import FileDownloader
+from .text_extractor import TextExtractor
+from .text_chunker import TextChunker
+from .embedding_manager import EmbeddingManager
+from .vector_storage import VectorStorage
+from .metadata_manager import MetadataManager
+
+# Import new extractors
+from .docx_extractor import extract_docx
+from .pptx_extractor import extract_pptx
+from .xlsx_extractor import extract_xlsx
+from .image_extractor import extract_image_content
+
+
+class ModularDocumentPreprocessor:
+    """
+    Modular document preprocessor that orchestrates the entire preprocessing pipeline.
+    
+    This class combines all preprocessing modules to provide a clean interface
+    for document processing while maintaining separation of concerns.
+    """
+    
+    def __init__(self):
+        """Initialize the modular document preprocessor."""
+        # Set up base database path
+        self.base_db_path = Path(OUTPUT_DIR).resolve()
+        self._ensure_base_directory()
+        
+        # Initialize all modules
+        self.pdf_downloader = PDFDownloader()  # Keep for backward compatibility
+        self.file_downloader = FileDownloader()  # New enhanced downloader
+        self.text_extractor = TextExtractor()
+        self.text_chunker = TextChunker()
+        self.embedding_manager = EmbeddingManager()
+        self.vector_storage = VectorStorage(self.base_db_path)
+        self.metadata_manager = MetadataManager(self.base_db_path)
+        
+        print("✅ Modular Document Preprocessor initialized successfully")
+    
+    def _ensure_base_directory(self):
+        """Ensure the base directory exists."""
+        if not self.base_db_path.exists():
+            try:
+                self.base_db_path.mkdir(parents=True, exist_ok=True)
+                print(f"✅ Created directory: {self.base_db_path}")
+            except PermissionError:
+                print(f"⚠️  Directory {self.base_db_path} should exist in production environment")
+                if not self.base_db_path.exists():
+                    raise RuntimeError(f"Required directory {self.base_db_path} does not exist and cannot be created")
+    
+    # Delegate metadata operations to metadata manager
+    def generate_doc_id(self, document_url: str) -> str:
+        """Generate a unique document ID from the URL."""
+        return self.metadata_manager.generate_doc_id(document_url)
+    
+    def is_document_processed(self, document_url: str) -> bool:
+        """Check if a document has already been processed."""
+        return self.metadata_manager.is_document_processed(document_url)
+    
+    def get_document_info(self, document_url: str) -> Dict[str, Any]:
+        """Get information about a processed document."""
+        return self.metadata_manager.get_document_info(document_url)
+    
+    def list_processed_documents(self) -> Dict[str, Dict]:
+        """List all processed documents."""
+        return self.metadata_manager.list_processed_documents()
+    
+    def get_collection_stats(self) -> Dict[str, Any]:
+        """Get statistics about all collections."""
+        return self.metadata_manager.get_collection_stats()
+    
+    async def process_document(self, document_url: str, force_reprocess: bool = False, timeout: int = 300) -> Union[str, List]:
+        """
+        Process a single document: download, extract, chunk, embed, and store.
+        
+        Args:
+            document_url: URL of the document (PDF, DOCX, PPTX, XLSX, images, etc.)
+            force_reprocess: If True, reprocess even if already processed
+            timeout: Download timeout in seconds (default: 300s/5min)
+            
+        Returns:
+            str: Document ID for normal processing
+            List: [content, type] for special handling (oneshot, tabular, image)
+        """
+        doc_id = self.generate_doc_id(document_url)
+        
+        # Check if already processed
+        if not force_reprocess and self.is_document_processed(document_url):
+            print(f"✅ Document {doc_id} already processed, skipping...")
+            return doc_id
+        
+        print(f"🚀 Processing document: {doc_id}")
+        print(f"📄 URL: {document_url}")
+        
+        temp_file_path = None
+        try:
+            # Step 1: Download file (enhanced to handle multiple types)
+            temp_file_path, ext = await self.file_downloader.download_file(document_url, timeout=timeout)
+            
+            if temp_file_path == 'not supported':
+                return ['unsupported', ext]
+
+            # Step 2: Extract text based on file type
+            full_text = ""
+            match ext:
+                case 'pdf':
+                    full_text = await self.text_extractor.extract_text_from_pdf(temp_file_path)
+                
+                case 'docx':
+                    full_text = extract_docx(temp_file_path)
+                
+                case 'pptx':
+                    full_text = extract_pptx(temp_file_path)
+                    return [full_text, 'oneshot']
+                
+                case 'url':
+                    new_context = "URL for Context: " + temp_file_path
+                    return [new_context, 'oneshot']
+                
+                case 'txt':
+                    with open(temp_file_path, 'r', encoding='utf-8') as f:
+                        full_text = f.read()
+                
+                case 'xlsx':
+                    full_text = extract_xlsx(temp_file_path)
+                    # Print a short preview (10-15 chars) to verify extraction
+                    try:
+                        preview = ''.join(full_text.split())[:15]
+                        if preview:
+                            print(f"🔎 XLSX extracted preview: {preview}")
+                    except Exception:
+                        pass
+                    return [full_text, 'tabular']
+                
+                case 'csv':
+                    with open(temp_file_path, 'r', encoding='utf-8') as f:
+                        full_text = f.read()
+                    return [full_text, 'tabular']
+
+                case 'png' | 'jpeg' | 'jpg':
+                    # Don't clean up image files - they'll be cleaned up by the caller
+                    return [temp_file_path, 'image', True]  # Third element indicates no cleanup needed
+                
+                case _:
+                    raise Exception(f"Unsupported file type: {ext}")
+
+            # Validate extracted text
+            if not self.text_extractor.validate_extracted_text(full_text):
+                raise Exception("No meaningful text extracted from document")
+            
+            # Step 3: Create chunks
+            chunks = self.text_chunker.chunk_text(full_text)
+            
+            # Check if document is too short for chunking
+            if len(chunks) < 5:
+                print(f"Only {len(chunks)} chunks formed, going for oneshot.")
+                return [full_text, 'oneshot']
+            
+            if not chunks:
+                raise Exception("No chunks created from text")
+            
+            # Log chunk statistics
+            chunk_stats = self.text_chunker.get_chunk_stats(chunks)
+            print(f"📊 Chunk Statistics: {chunk_stats['total_chunks']} chunks, "
+                  f"avg size: {chunk_stats['avg_chunk_size']:.0f} chars")
+            
+            # Step 4: Create embeddings
+            embeddings = await self.embedding_manager.create_embeddings(chunks)
+            
+            # Validate embeddings
+            if not self.embedding_manager.validate_embeddings(embeddings, len(chunks)):
+                raise Exception("Invalid embeddings generated")
+            
+            # Step 5: Store in Qdrant
+            await self.vector_storage.store_in_qdrant(chunks, embeddings, doc_id)
+            
+            # Step 6: Save metadata
+            self.metadata_manager.save_document_metadata(chunks, doc_id, document_url)
+            
+            print(f"✅ Document {doc_id} processed successfully: {len(chunks)} chunks")
+            return doc_id
+            
+        except Exception as e:
+            print(f"❌ Error processing document {doc_id}: {str(e)}")
+            raise
+        finally:
+            # Clean up temporary file - but NOT for images since they need the file path
+            if temp_file_path and ext not in ['png', 'jpeg', 'jpg']:
+                self.file_downloader.cleanup_temp_file(temp_file_path)
+    
+    async def process_multiple_documents(self, document_urls: List[str], force_reprocess: bool = False) -> Dict[str, str]:
+        """
+        Process multiple documents concurrently.
+        
+        Args:
+            document_urls: List of PDF URLs
+            force_reprocess: If True, reprocess even if already processed
+            
+        Returns:
+            Dict[str, str]: Mapping of URLs to document IDs
+        """
+        print(f"🚀 Processing {len(document_urls)} documents...")
+        
+        results = {}
+        
+        # Process documents concurrently (with limited concurrency)
+        semaphore = asyncio.Semaphore(3)  # Limit to 3 concurrent downloads
+        
+        async def process_single(url):
+            async with semaphore:
+                try:
+                    doc_id = await self.process_document(url, force_reprocess)
+                    return url, doc_id
+                except Exception as e:
+                    print(f"❌ Failed to process {url}: {str(e)}")
+                    return url, None
+        
+        tasks = [process_single(url) for url in document_urls]
+        completed_tasks = await asyncio.gather(*tasks, return_exceptions=True)
+        
+        for result in completed_tasks:
+            if isinstance(result, tuple):
+                url, doc_id = result
+                if doc_id:
+                    results[url] = doc_id
+        
+        print(f"✅ Successfully processed {len(results)}/{len(document_urls)} documents")
+        return results
+    
+    def get_system_info(self) -> Dict[str, Any]:
+        """
+        Get information about the preprocessing system.
+        
+        Returns:
+            Dict[str, Any]: System information
+        """
+        return {
+            "base_db_path": str(self.base_db_path),
+            "embedding_model": self.embedding_manager.get_model_info(),
+            "text_chunker_config": {
+                "chunk_size": self.text_chunker.chunk_size,
+                "chunk_overlap": self.text_chunker.chunk_overlap
+            },
+            "processed_documents_registry": self.metadata_manager.get_registry_path(),
+            "collection_stats": self.get_collection_stats()
+        }
+    
+    def cleanup_document(self, document_url: str) -> bool:
+        """
+        Remove all data for a specific document.
+        
+        Args:
+            document_url: URL of the document to clean up
+            
+        Returns:
+            bool: True if successfully cleaned up
+        """
+        doc_id = self.generate_doc_id(document_url)
+        
+        try:
+            # Remove vector storage
+            vector_removed = self.vector_storage.delete_collection(doc_id)
+            
+            # Remove metadata
+            metadata_removed = self.metadata_manager.remove_document_metadata(doc_id)
+            
+            success = vector_removed and metadata_removed
+            if success:
+                print(f"✅ Successfully cleaned up document {doc_id}")
+            else:
+                print(f"⚠️ Partial cleanup for document {doc_id}")
+            
+            return success
+            
+        except Exception as e:
+            print(f"❌ Error cleaning up document {doc_id}: {e}")
+            return False

preprocessing/preprocessing_modules/pdf_downloader.py

@@ -0,0 +1,112 @@
+"""
+PDF Downloader Module
+
+Handles downloading PDFs from URLs with retry logic and progress tracking.
+"""
+
+import os
+import asyncio
+import tempfile
+import aiohttp
+from typing import Optional
+
+
+class PDFDownloader:
+    """Handles PDF downloading with enhanced error handling and retry logic."""
+    
+    def __init__(self):
+        """Initialize the PDF downloader."""
+        pass
+    
+    async def download_pdf(self, url: str, timeout: int = 300, max_retries: int = 3) -> str:
+        """
+        Download PDF from URL to a temporary file with enhanced error handling.
+        
+        Args:
+            url: URL of the PDF to download
+            timeout: Download timeout in seconds (default: 300s/5min)
+            max_retries: Maximum number of retry attempts
+            
+        Returns:
+            str: Path to the downloaded temporary file
+            
+        Raises:
+            Exception: If download fails after all retries
+        """
+        print(f"📥 Downloading PDF from: {url[:50]}...")
+        
+        for attempt in range(max_retries):
+            try:
+                # Enhanced timeout settings for large files
+                timeout_config = aiohttp.ClientTimeout(
+                    total=timeout,          # Total timeout
+                    connect=30,             # Connection timeout
+                    sock_read=120           # Socket read timeout
+                )
+                
+                async with aiohttp.ClientSession(timeout=timeout_config) as session:
+                    print(f"   Attempt {attempt + 1}/{max_retries} (timeout: {timeout}s)")
+                    
+                    async with session.get(url) as response:
+                        if response.status != 200:
+                            raise Exception(f"Failed to download PDF: HTTP {response.status}")
+                        
+                        # Get content length for progress tracking
+                        content_length = response.headers.get('content-length')
+                        if content_length:
+                            total_size = int(content_length)
+                            print(f"   File size: {total_size / (1024*1024):.1f} MB")
+                        
+                        # Create temporary file
+                        temp_file = tempfile.NamedTemporaryFile(
+                            delete=False, 
+                            suffix=".pdf",
+                            prefix="preprocess_"
+                        )
+                        
+                        # Write content to temporary file with progress tracking
+                        downloaded = 0
+                        async for chunk in response.content.iter_chunked(16384):  # Larger chunks
+                            temp_file.write(chunk)
+                            downloaded += len(chunk)
+                            
+                            # Show progress for large files
+                            if content_length and downloaded % (1024*1024) == 0:  # Every MB
+                                progress = (downloaded / total_size) * 100
+                                print(f"   Progress: {progress:.1f}% ({downloaded/(1024*1024):.1f} MB)")
+                        
+                        temp_file.close()
+                        print(f"✅ PDF downloaded successfully: {temp_file.name}")
+                        return temp_file.name
+                        
+            except asyncio.TimeoutError:
+                print(f"   ⏰ Timeout on attempt {attempt + 1}")
+                if attempt < max_retries - 1:
+                    wait_time = (attempt + 1) * 30  # Increasing wait time
+                    print(f"   ⏳ Waiting {wait_time}s before retry...")
+                    await asyncio.sleep(wait_time)
+                continue
+                
+            except Exception as e:
+                print(f"   ❌ Error on attempt {attempt + 1}: {str(e)}")
+                if attempt < max_retries - 1:
+                    wait_time = (attempt + 1) * 15
+                    print(f"   ⏳ Waiting {wait_time}s before retry...")
+                    await asyncio.sleep(wait_time)
+                continue
+        
+        raise Exception(f"Failed to download PDF after {max_retries} attempts")
+    
+    def cleanup_temp_file(self, temp_path: str) -> None:
+        """
+        Clean up temporary file.
+        
+        Args:
+            temp_path: Path to the temporary file to delete
+        """
+        if temp_path and os.path.exists(temp_path):
+            try:
+                os.unlink(temp_path)
+                print(f"🗑️ Cleaned up temporary file: {temp_path}")
+            except Exception as e:
+                print(f"⚠️ Warning: Could not delete temporary file {temp_path}: {e}")

preprocessing/preprocessing_modules/pptx_extractor.py

@@ -0,0 +1,118 @@
+from pptx import Presentation
+from pptx.enum.shapes import MSO_SHAPE_TYPE
+from typing import List, Dict, Any
+from PIL import Image
+from io import BytesIO
+import requests
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import tempfile
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
+from config import config
+
+# OCR Space API configuration
+API_KEY = getattr(config, 'OCR_SPACE_API_KEY', None)
+API_URL = "https://api.ocr.space/parse/image"
+
+def ocr_space_file(filename, api_key=API_KEY, overlay=False, language="eng"):
+    """Extract text from image file using OCR Space API"""
+    if not api_key:
+        return filename, "OCR API key not configured"
+        
+    payload = {
+        "isOverlayRequired": overlay,
+        "apikey": api_key,
+        "language": language,
+        "detectOrientation": True,
+        "scale": True,
+        "isTable": False,
+        "OCREngine": 2
+    }
+    try:
+        with open(filename, "rb") as f:
+            response = requests.post(API_URL, files={filename: f}, data=payload, timeout=30)
+        
+        if response.status_code != 200:
+            return filename, f"API Error: HTTP {response.status_code}"
+            
+        parsed = response.json()
+        
+        if parsed.get("OCRExitCode") == 1:
+            parsed_text = parsed.get("ParsedResults", [{}])[0].get("ParsedText", "")
+            return filename, parsed_text
+        else:
+            error_msg = parsed.get("ErrorMessage", ["Unknown error"])[0] if parsed.get("ErrorMessage") else "Unknown OCR error"
+            return filename, f"OCR Error: {error_msg}"
+            
+    except requests.exceptions.Timeout:
+        return filename, "Error: Request timeout"
+    except requests.exceptions.RequestException as e:
+        return filename, f"Error: Network error - {str(e)}"
+    except Exception as e:
+        return filename, f"Error: {e}"
+
+def extract_pptx(pptx_path: str) -> str:
+    """Extract text and images from PowerPoint presentations."""
+    try:
+        prs = Presentation(pptx_path)
+    except Exception as e:
+        return f"Error loading PowerPoint file: {str(e)}"
+
+    all_content = []
+    temp_files = []
+
+    try:
+        for slide_idx, slide in enumerate(prs.slides):
+            slide_content = [f"\\n=== Slide {slide_idx + 1} ===\\n"]
+            slide_images = []
+
+            for shape in slide.shapes:
+                # Extract text
+                if hasattr(shape, "text") and shape.text.strip():
+                    slide_content.append(shape.text.strip())
+                
+                # Extract images
+                elif shape.shape_type == MSO_SHAPE_TYPE.PICTURE:
+                    try:
+                        image = shape.image
+                        image_bytes = image.blob
+                        
+                        # Save image to temp file
+                        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
+                        temp_file.write(image_bytes)
+                        temp_file.close()
+                        temp_files.append(temp_file.name)
+                        slide_images.append(temp_file.name)
+                    except Exception as e:
+                        slide_content.append(f"[Image extraction error: {str(e)}]")
+
+            # Process images with OCR if API key is available
+            if slide_images and API_KEY:
+                try:
+                    with ThreadPoolExecutor(max_workers=3) as executor:
+                        future_to_filename = {
+                            executor.submit(ocr_space_file, img_file): img_file 
+                            for img_file in slide_images
+                        }
+                        
+                        for future in as_completed(future_to_filename):
+                            filename, ocr_result = future.result()
+                            if ocr_result and not ocr_result.startswith("Error") and not ocr_result.startswith("OCR Error"):
+                                slide_content.append(f"[Image Text]: {ocr_result}")
+                except Exception as e:
+                    slide_content.append(f"[OCR processing error: {str(e)}]")
+            elif slide_images:
+                slide_content.append(f"[{len(slide_images)} images found - OCR not available]")
+
+            all_content.append("\\n".join(slide_content))
+
+    finally:
+        # Clean up temp files
+        for temp_file in temp_files:
+            try:
+                os.unlink(temp_file)
+            except:
+                pass
+
+    return "\\n\\n".join(all_content)

preprocessing/preprocessing_modules/text_chunker.py

@@ -0,0 +1,167 @@
+"""
+Text Chunker Module
+
+Handles chunking text into smaller pieces with overlap for better context preservation.
+"""
+
+import re
+from typing import List
+from config.config import CHUNK_SIZE, CHUNK_OVERLAP
+
+
+class TextChunker:
+    """Handles text chunking with overlap and smart boundary detection."""
+    
+    def __init__(self):
+        """Initialize the text chunker."""
+        self.chunk_size = CHUNK_SIZE
+        self.chunk_overlap = CHUNK_OVERLAP
+    
+    def chunk_text(self, text: str) -> List[str]:
+        """
+        Chunk text into smaller pieces with overlap.
+        
+        Args:
+            text: The input text to chunk
+            
+        Returns:
+            List[str]: List of text chunks
+        """
+        print(f"✂️ Chunking text into {self.chunk_size} character chunks with {self.chunk_overlap} overlap")
+        
+        # Clean the text
+        cleaned_text = self._clean_text(text)
+        
+        chunks = []
+        start = 0
+        
+        while start < len(cleaned_text):
+            end = start + self.chunk_size
+            
+            # Try to end at sentence boundary
+            if end < len(cleaned_text):
+                end = self._find_sentence_boundary(cleaned_text, start, end)
+            
+            chunk = cleaned_text[start:end].strip()
+            
+            # Only add chunk if it's meaningful
+            if chunk and len(chunk) > 50:
+                chunks.append(chunk)
+            
+            # Move start position with overlap
+            start = end - self.chunk_overlap
+            if start >= len(cleaned_text):
+                break
+        
+        print(f"✅ Created {len(chunks)} chunks (size={self.chunk_size}, overlap={self.chunk_overlap})")
+        return chunks
+    
+    def _clean_text(self, text: str) -> str:
+        """
+        Clean text by normalizing whitespace and removing excessive line breaks.
+        
+        Args:
+            text: Raw text to clean
+            
+        Returns:
+            str: Cleaned text
+        """
+        # Replace multiple whitespace with single space
+        text = re.sub(r'\s+', ' ', text)
+        return text.strip()
+    
+    def _find_sentence_boundary(self, text: str, start: int, preferred_end: int) -> int:
+        """
+        Find the best sentence boundary near the preferred end position.
+        
+        Args:
+            text: The full text
+            start: Start position of the chunk
+            preferred_end: Preferred end position
+            
+        Returns:
+            int: Adjusted end position at sentence boundary
+        """
+        # Look for sentence endings within a reasonable range
+        search_start = max(start, preferred_end - 100)
+        search_end = min(len(text), preferred_end + 50)
+        
+        sentence_endings = ['.', '!', '?']
+        best_end = preferred_end
+        
+        # Search backwards from preferred end for sentence boundary
+        for i in range(preferred_end - 1, search_start - 1, -1):
+            if text[i] in sentence_endings:
+                # Check if this looks like a real sentence ending
+                if self._is_valid_sentence_ending(text, i):
+                    best_end = i + 1
+                    break
+        
+        return best_end
+    
+    def _is_valid_sentence_ending(self, text: str, pos: int) -> bool:
+        """
+        Check if a punctuation mark represents a valid sentence ending.
+        
+        Args:
+            text: The full text
+            pos: Position of the punctuation mark
+            
+        Returns:
+            bool: True if it's a valid sentence ending
+        """
+        # Avoid breaking on abbreviations like "Dr.", "Mr.", etc.
+        if pos > 0 and text[pos] == '.':
+            # Look at the character before the period
+            char_before = text[pos - 1]
+            if char_before.isupper():
+                # Might be an abbreviation
+                word_start = pos - 1
+                while word_start > 0 and text[word_start - 1].isalpha():
+                    word_start -= 1
+                
+                word = text[word_start:pos]
+                # Common abbreviations to avoid breaking on
+                abbreviations = {'Dr', 'Mr', 'Mrs', 'Ms', 'Prof', 'Inc', 'Ltd', 'Corp', 'Co'}
+                if word in abbreviations:
+                    return False
+        
+        # Check if there's a space or newline after the punctuation
+        if pos + 1 < len(text):
+            next_char = text[pos + 1]
+            return next_char.isspace() or next_char.isupper()
+        
+        return True
+    
+    def get_chunk_stats(self, chunks: List[str]) -> dict:
+        """
+        Get statistics about the created chunks.
+        
+        Args:
+            chunks: List of text chunks
+            
+        Returns:
+            dict: Statistics about the chunks
+        """
+        if not chunks:
+            return {
+                "total_chunks": 0,
+                "total_characters": 0,
+                "total_words": 0,
+                "avg_chunk_size": 0,
+                "min_chunk_size": 0,
+                "max_chunk_size": 0
+            }
+        
+        chunk_sizes = [len(chunk) for chunk in chunks]
+        total_chars = sum(chunk_sizes)
+        total_words = sum(len(chunk.split()) for chunk in chunks)
+        
+        return {
+            "total_chunks": len(chunks),
+            "total_characters": total_chars,
+            "total_words": total_words,
+            "avg_chunk_size": total_chars / len(chunks),
+            "min_chunk_size": min(chunk_sizes),
+            "max_chunk_size": max(chunk_sizes)
+        }

preprocessing/preprocessing_modules/text_extractor.py

@@ -0,0 +1,62 @@
+"""
+Text Extractor Module
+
+Handles extracting text content from PDF files.
+"""
+
+import pdfplumber
+
+
+class TextExtractor:
+    """Handles text extraction from PDF files."""
+    
+    def __init__(self):
+        """Initialize the text extractor."""
+        pass
+    
+    async def extract_text_from_pdf(self, pdf_path: str) -> str:
+        """
+        Extract text from PDF file.
+        
+        Args:
+            pdf_path: Path to the PDF file
+            
+        Returns:
+            str: Extracted text content
+            
+        Raises:
+            Exception: If text extraction fails
+        """
+        print(f"📖 Extracting text from PDF...")
+        
+        full_text = ""
+        try:
+            with pdfplumber.open(pdf_path) as pdf:
+                for page_num, page in enumerate(pdf.pages):
+                    text = page.extract_text()
+                    if text:
+                        full_text += f"\n--- Page {page_num + 1} ---\n"
+                        full_text += text
+            
+            print(f"✅ Extracted {len(full_text)} characters from PDF")
+            return full_text
+            
+        except Exception as e:
+            raise Exception(f"Failed to extract text from PDF: {str(e)}")
+    
+    def validate_extracted_text(self, text: str) -> bool:
+        """
+        Validate that extracted text is not empty and contains meaningful content.
+        
+        Args:
+            text: The extracted text to validate
+            
+        Returns:
+            bool: True if text is valid, False otherwise
+        """
+        if not text or not text.strip():
+            return False
+        
+        # Check if text has at least some alphabetic characters
+        alphabetic_chars = sum(1 for char in text if char.isalpha())
+        return alphabetic_chars > 50  # At least 50 alphabetic characters

preprocessing/preprocessing_modules/vector_storage.py

@@ -0,0 +1,212 @@
+"""
+Vector Storage Module
+
+Handles storing chunks and embeddings in Qdrant vector database.
+"""
+
+import numpy as np
+from typing import List
+from pathlib import Path
+from qdrant_client import QdrantClient
+from qdrant_client.models import Distance, VectorParams, PointStruct
+
+
+class VectorStorage:
+    """Handles vector storage operations with Qdrant."""
+    
+    def __init__(self, base_db_path: Path):
+        """
+        Initialize the vector storage.
+        
+        Args:
+            base_db_path: Base path for storing Qdrant databases
+        """
+        self.base_db_path = base_db_path
+    
+    async def store_in_qdrant(self, chunks: List[str], embeddings: np.ndarray, doc_id: str):
+        """
+        Store chunks and embeddings in Qdrant.
+        
+        Args:
+            chunks: List of text chunks
+            embeddings: Corresponding embeddings array
+            doc_id: Document identifier
+        """
+        if len(chunks) != embeddings.shape[0]:
+            raise ValueError(f"Chunk count ({len(chunks)}) doesn't match embedding count ({embeddings.shape[0]})")
+        
+        collection_name = f"{doc_id}_collection"
+        db_path = self.base_db_path / f"{collection_name}.db"
+        client = QdrantClient(path=str(db_path))
+        
+        print(f"💾 Storing {len(chunks)} vectors in collection: {collection_name}")
+        
+        try:
+            # Create or recreate collection
+            await self._setup_collection(client, collection_name, embeddings.shape[1])
+            
+            # Prepare and upload points
+            await self._upload_points(client, collection_name, chunks, embeddings, doc_id)
+            
+            print(f"✅ Successfully stored all vectors in Qdrant")
+            
+        finally:
+            client.close()
+    
+    async def _setup_collection(self, client: QdrantClient, collection_name: str, embedding_dim: int):
+        """
+        Set up Qdrant collection, recreating if it exists.
+        
+        Args:
+            client: Qdrant client
+            collection_name: Name of the collection
+            embedding_dim: Dimension of embeddings
+        """
+        # Delete existing collection if it exists
+        try:
+            client.delete_collection(collection_name)
+            print(f"🗑️ Deleted existing collection: {collection_name}")
+        except Exception:
+            pass  # Collection might not exist
+        
+        # Create new collection
+        client.create_collection(
+            collection_name=collection_name,
+            vectors_config=VectorParams(
+                size=embedding_dim,
+                distance=Distance.COSINE
+            )
+        )
+        print(f"✅ Created new collection: {collection_name}")
+    
+    async def _upload_points(self, client: QdrantClient, collection_name: str, 
+                           chunks: List[str], embeddings: np.ndarray, doc_id: str):
+        """
+        Upload points to Qdrant collection in batches.
+        
+        Args:
+            client: Qdrant client
+            collection_name: Name of the collection
+            chunks: Text chunks
+            embeddings: Embedding vectors
+            doc_id: Document identifier
+        """
+        # Prepare points
+        points = []
+        for i in range(len(chunks)):
+            points.append(
+                PointStruct(
+                    id=i,
+                    vector=embeddings[i].tolist(),
+                    payload={
+                        "text": chunks[i],
+                        "chunk_id": i,
+                        "doc_id": doc_id,
+                        "char_count": len(chunks[i]),
+                        "word_count": len(chunks[i].split())
+                    }
+                )
+            )
+        
+        # Upload in batches to handle large documents
+        batch_size = 100
+        total_batches = (len(points) + batch_size - 1) // batch_size
+        
+        for i in range(0, len(points), batch_size):
+            batch = points[i:i + batch_size]
+            batch_num = (i // batch_size) + 1
+            
+            print(f"   Uploading batch {batch_num}/{total_batches} ({len(batch)} points)")
+            client.upsert(collection_name=collection_name, points=batch)
+        
+        print(f"✅ Uploaded {len(points)} points in {total_batches} batches")
+    
+    def collection_exists(self, doc_id: str) -> bool:
+        """
+        Check if a collection exists for the given document ID.
+        
+        Args:
+            doc_id: Document identifier
+            
+        Returns:
+            bool: True if collection exists, False otherwise
+        """
+        collection_name = f"{doc_id}_collection"
+        db_path = self.base_db_path / f"{collection_name}.db"
+        return db_path.exists()
+    
+    def get_collection_info(self, doc_id: str) -> dict:
+        """
+        Get information about a collection.
+        
+        Args:
+            doc_id: Document identifier
+            
+        Returns:
+            dict: Collection information
+        """
+        collection_name = f"{doc_id}_collection"
+        db_path = self.base_db_path / f"{collection_name}.db"
+        
+        if not db_path.exists():
+            return {
+                "collection_name": collection_name,
+                "exists": False,
+                "path": str(db_path)
+            }
+        
+        try:
+            client = QdrantClient(path=str(db_path))
+            try:
+                collection_info = client.get_collection(collection_name)
+                return {
+                    "collection_name": collection_name,
+                    "exists": True,
+                    "path": str(db_path),
+                    "vectors_count": collection_info.vectors_count,
+                    "status": collection_info.status
+                }
+            finally:
+                client.close()
+        except Exception as e:
+            return {
+                "collection_name": collection_name,
+                "exists": True,
+                "path": str(db_path),
+                "error": str(e)
+            }
+    
+    def delete_collection(self, doc_id: str) -> bool:
+        """
+        Delete a collection and its database file.
+        
+        Args:
+            doc_id: Document identifier
+            
+        Returns:
+            bool: True if successfully deleted, False otherwise
+        """
+        collection_name = f"{doc_id}_collection"
+        db_path = self.base_db_path / f"{collection_name}.db"
+        
+        try:
+            if db_path.exists():
+                # Try to delete collection properly first
+                try:
+                    client = QdrantClient(path=str(db_path))
+                    client.delete_collection(collection_name)
+                    client.close()
+                except Exception:
+                    pass  # Collection might not exist or be corrupted
+                
+                # Remove database directory
+                import shutil
+                shutil.rmtree(db_path, ignore_errors=True)
+                print(f"🗑️ Deleted collection: {collection_name}")
+                return True
+            
+        except Exception as e:
+            print(f"❌ Error deleting collection {collection_name}: {e}")
+            return False
+        
+        return True  # Nothing to delete

preprocessing/preprocessing_modules/xlsx_extractor.py

@@ -0,0 +1,119 @@
+from openpyxl import load_workbook
+from openpyxl.drawing.image import Image as OpenPyXLImage
+from typing import List, Dict, Any
+from PIL import Image
+from io import BytesIO
+import pytesseract
+import os
+import pandas as pd
+
+def extract_xlsx(xlsx_path: str, tesseract_cmd: str = None) -> str:
+    """Extract data from Excel files including text and images."""
+    if tesseract_cmd:
+        pytesseract.pytesseract.tesseract_cmd = tesseract_cmd
+
+    try:
+        wb = load_workbook(xlsx_path, data_only=True)
+    except Exception as e:
+        return f"Error loading Excel file: {str(e)}"
+
+    all_sheets_content: list[str] = []
+    preview_text: str | None = None
+    any_data_found = False
+
+    for sheet in wb.worksheets:
+        sheet_content = [f"\n=== Sheet: {sheet.title} ===\n"]
+
+        # Extract table data
+        has_data = False
+        non_empty_rows = 0
+        for row in sheet.iter_rows(max_row=sheet.max_row, values_only=True):
+            if row is None or all(cell is None for cell in row):
+                continue  # skip completely empty rows
+            has_data = True
+            non_empty_rows += 1
+            any_data_found = True
+            row_data = [str(cell).strip() if cell is not None else "" for cell in row]
+            joined = " | ".join(row_data)
+            sheet_content.append(joined)
+            if preview_text is None and joined.strip():
+                preview_text = joined[:15]
+
+        if not has_data:
+            sheet_content.append("[No data in this sheet]")
+            print(f"ℹ️ XLSX: Sheet '{sheet.title}' has no data (openpyxl)")
+        else:
+            print(f"🧾 XLSX: Sheet '{sheet.title}' non-empty rows: {non_empty_rows}")
+
+        # Extract images from the sheet
+        if hasattr(sheet, '_images'):
+            image_count = 0
+            for img in sheet._images:
+                try:
+                    if hasattr(img, '_data'):  # if it's a real OpenPyXL Image
+                        image_data = img._data()
+                    elif hasattr(img, '_ref'):
+                        continue  # cell ref-only images; ignore
+                    else:
+                        continue
+
+                    pil_img = Image.open(BytesIO(image_data))
+                    try:
+                        ocr_text = pytesseract.image_to_string(pil_img).strip()
+                        if ocr_text:
+                            sheet_content.append(f"[Image {image_count + 1} Text]: {ocr_text}")
+                        else:
+                            sheet_content.append(f"[Image {image_count + 1}]: No text detected")
+                    except Exception as ocr_e:
+                        sheet_content.append(f"[Image {image_count + 1}]: OCR failed - {str(ocr_e)}")
+
+                    image_count += 1
+                except Exception as e:
+                    sheet_content.append(f"[Image extraction error: {str(e)}]")
+
+            if image_count == 0:
+                sheet_content.append("[No images found in this sheet]")
+
+        all_sheets_content.append("\n".join(sheet_content))
+
+    # If no data found using openpyxl, try pandas fallback (handles some edge cases better)
+    if not any_data_found:
+        print("ℹ️ XLSX: No data via openpyxl, trying pandas fallback…")
+        try:
+            xls = pd.ExcelFile(xlsx_path, engine="openpyxl")
+            pandas_parts = []
+            extracted_sheets = 0
+            for sheet_name in xls.sheet_names:
+                df = pd.read_excel(xls, sheet_name=sheet_name, dtype=str)
+                if not df.empty:
+                    any_data_found = True
+                    header = f"\n=== Sheet: {sheet_name} ===\n"
+                    csv_like = df.fillna("").astype(str).to_csv(index=False)
+                    pandas_parts.append(header + csv_like)
+                    extracted_sheets += 1
+                    if preview_text is None:
+                        flat = "".join(csv_like.splitlines())
+                        if flat:
+                            preview_text = flat[:15]
+                else:
+                    pandas_parts.append(f"\n=== Sheet: {sheet_name} ===\n[No data in this sheet]")
+            if pandas_parts:
+                all_sheets_content = pandas_parts
+            print(f"✅ XLSX: Pandas fallback extracted {extracted_sheets} non-empty sheet(s)")
+        except Exception as pe:
+            # If pandas also fails, keep whatever we had
+            all_sheets_content.append(f"[Pandas fallback failed: {str(pe)}]")
+            print(f"❌ XLSX: Pandas fallback failed: {pe}")
+
+    combined = "\n\n".join(all_sheets_content)
+
+    # Print a small preview for verification
+    if preview_text is None:
+        # fallback: take from combined text
+        flat_combined = "".join(combined.splitlines()).strip()
+        if flat_combined:
+            preview_text = flat_combined[:15]
+    if preview_text:
+        print(f"🔎 XLSX content preview: {preview_text}")
+
+    return combined

requirements.txt

@@ -0,0 +1,33 @@
+fastapi
+uvicorn
+pydantic
+aiohttp
+pdfplumber
+sentence-transformers
+qdrant-client
+openai
+google-generativeai
+numpy
+tqdm
+python-multipart
+jinja2
+python-dotenv
+rank-bm25
+transformers
+torch
+
+# New dependencies for multiple file format support
+python-docx
+python-pptx
+openpyxl
+pillow
+pytesseract
+opencv-python
+pandas
+beautifulsoup4
+lxml
+langchain-groq
+langchain-google-genai
+langchain-core
+httpx
+groq