created doc-mcp

.env.example

@@ -0,0 +1,3 @@
+GITHUB_API_KEY=
+NEBIUS_API_KEY=
+MONGODB_URI=

.gitignore

@@ -0,0 +1,3 @@
+.env
+.venv
+__pycache__

.python-version

@@ -0,0 +1 @@
+3.13

README.md

@@ -5,10 +5,104 @@ colorFrom: yellow
 colorTo: pink
 sdk: gradio
 sdk_version: 5.33.0
+python_version: 3.13
 app_file: app.py
 pinned: false
 license: mit
 short_description: 'RAG on documentations for your agent '
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Doc-MCP 📚
+
+> Transform GitHub documentation repositories into accessible MCP (Model Context Protocol) servers for AI agents
+
+**Hackathon Track**: `mcp-server-track`
+
+## 🎯 What is Doc-MCP?
+
+Doc-MCP ingests markdown documentation from GitHub repositories and creates MCP servers that provide easy access to documentation context for AI agents. Just point it at any GitHub repo with markdown docs, and get an intelligent Q&A interface powered by vector search.
+
+## ✨ Key Features
+
+- **GitHub Integration**: Fetch markdown files directly from any GitHub repository
+- **Vector Search**: Uses MongoDB Atlas with Nebius AI embeddings for semantic search
+- **MCP Server**: Exposes documentation as MCP endpoints for AI agents
+- **Smart Q&A**: Ask questions about documentation with source citations
+- **Repository Management**: Track multiple repositories and their statistics
+
+## 🚀 Quick Start
+
+1. **Setup Environment**:
+```bash
+# Clone and install
+git clone https://github.com/yourusername/doc-mcp.git
+cd doc-mcp
+uv sync
+
+# Configure environment
+cp .env.example .env
+# Add your NEBIUS_API_KEY and MONGODB_URI
+```
+
+2. **Run the App**:
+```bash
+python main.py
+# Open http://localhost:7860
+```
+
+3. **Ingest Documentation**:
+   - Enter a GitHub repo URL (e.g., `gradio-app/gradio`)
+   - Select markdown files to process
+   - Load files and generate vector embeddings
+
+4. **Query Documentation**:
+   - Select your repository
+   - Ask questions about the documentation
+   - Get answers with source citations
+
+
+## Workflow
+```mermaid
+flowchart TD
+ subgraph Ingestion["Ingestion"]
+        B["Discover Markdown Files"]
+        A["GitHub Repo URL"]
+        C["User File Selection"]
+        D["Chunk & Embed Documents"]
+        E["Store in MongoDB"]
+  end
+ subgraph Query["Query"]
+        G["Select Repository"]
+        F["User Question"]
+        H["Vector Search"]
+        I["Retrieve Context"]
+        J["Generate Response"]
+        K["Display with Sources"]
+  end
+    A --> B
+    B --> C
+    C --> D
+    D --> E
+    F --> G
+    G --> H
+    H --> I
+    I --> J
+    J --> K
+    E --> H
+```
+
+## 🛠️ Technology Stack
+
+- **Frontend**: Gradio
+- **Vector Store**: MongoDB Atlas with vector search
+- **Embeddings**: Nebius AI (BAAI/bge-en-icl)
+- **LLM**: Nebius LLM (Llama-3.3-70B-Instruct)
+- **Document Processing**: LlamaIndex
+
+## 📹 Demo Video
+
+[Link to demo video showing MCP server in action with Claude Desktop/Cursor]
+
+---
+
+**Transform your documentation into intelligent, accessible knowledge for AI agents!** 🚀

main.py

@@ -0,0 +1,1152 @@
+import asyncio
+import os
+import time
+from typing import Dict, List
+
+import gradio as gr
+from dotenv import load_dotenv
+from llama_index.core import Settings
+from llama_index.core.text_splitter import SentenceSplitter
+
+from rag.config import embed_model, get_available_repos, llm
+from rag.github_file_loader import \
+    fetch_markdown_files as fetch_files_with_loader, load_github_files
+from rag.ingest import ingest_documents_async
+
+load_dotenv()
+
+Settings.llm = llm
+Settings.embed_model = embed_model
+Settings.node_parser = SentenceSplitter(chunk_size=3072)
+
+
+def get_available_repositories():
+    return get_available_repos()
+    
+
+def start_file_loading(
+    repo_url: str, selected_files: List[str], current_progress: Dict
+):
+    """Step 1: Load files from GitHub"""
+    print("\n🔄 STARTING FILE LOADING STEP")
+    print(f"📍 Repository: {repo_url}")
+    print(f"📋 Selected files: {selected_files}")
+
+    if not selected_files:
+        return {
+            "status": "error",
+            "message": "❌ No files selected for loading",
+            "progress": 0,
+            "details": "",
+            "step": "file_loading",
+        }
+
+    total_files = len(selected_files)
+    start_time = time.time()
+
+    # Parse repo name from URL
+    if "github.com" in repo_url:
+        repo_name = (
+            repo_url.replace("https://github.com/", "")
+            .replace("http://github.com/", "")
+            .strip("/")
+        )
+        if "/" not in repo_name:
+            return {
+                "status": "error",
+                "message": "❌ Invalid repository URL format",
+                "progress": 0,
+                "details": "",
+                "step": "file_loading",
+            }
+    else:
+        repo_name = repo_url.strip()
+
+    try:
+        batch_size = 25
+        all_documents = []
+        all_failed = []
+
+        current_progress.update(
+            {
+                "status": "loading",
+                "message": f"🚀 Loading files from {repo_name}",
+                "progress": 0,
+                "total_files": total_files,
+                "processed_files": 0,
+                "phase": "File Loading",
+                "details": f"Processing {total_files} files in batches...",
+                "step": "file_loading",
+            }
+        )
+
+        for i in range(0, len(selected_files), batch_size):
+            batch = selected_files[i : i + batch_size]
+
+            print(f"\n📦 PROCESSING BATCH {i // batch_size + 1}")
+            print(f"   Files: {batch}")
+
+            # Update progress for current batch
+            progress_percentage = (i / total_files) * 100
+            current_progress.update(
+                {
+                    "progress": progress_percentage,
+                    "processed_files": i,
+                    "current_batch": i // batch_size + 1,
+                    "details": f"Loading batch {i // batch_size + 1}: {', '.join([f.split('/')[-1] for f in batch])}",
+                }
+            )
+
+            try:
+                documents, failed = load_github_files(
+                    repo_name=repo_name,
+                    file_paths=batch,
+                    branch="main",
+                    concurrent_requests=10,
+                    github_token=os.getenv("GITHUB_API_KEY"),
+                )
+
+                print("✅ Load results:")
+                print(f"   - Documents: {len(documents)}")
+                print(f"   - Failed: {len(failed)}")
+
+                if documents:
+                    for j, doc in enumerate(documents):
+                        print(f"   📄 Doc {j + 1}: {doc.doc_id}")
+                        print(f"      Size: {len(doc.text)} chars")
+
+                        # Ensure repo metadata is set
+                        if "repo" not in doc.metadata:
+                            doc.metadata["repo"] = repo_name
+                            print(f"      ✅ Added repo metadata: {repo_name}")
+
+                all_documents.extend(documents)
+                all_failed.extend(failed)
+
+            except Exception as batch_error:
+                print(f"❌ Batch processing error: {batch_error}")
+                all_failed.extend(batch)
+
+        loading_time = time.time() - start_time
+
+        # Store loaded documents in progress state for next step
+        current_progress.update(
+            {
+                "status": "loaded",
+                "message": f"✅ File Loading Complete! Loaded {len(all_documents)} documents",
+                "progress": 100,
+                "phase": "Files Loaded",
+                "details": f"Successfully loaded {len(all_documents)} documents in {loading_time:.1f}s",
+                "step": "file_loading_complete",
+                "loaded_documents": all_documents,  # Store documents for next step
+                "failed_files": all_failed,
+                "loading_time": loading_time,
+                "repo_name": repo_name,
+            }
+        )
+
+        return current_progress
+
+    except Exception as e:
+        total_time = time.time() - start_time
+        error_msg = f"❌ File loading error after {total_time:.1f}s: {str(e)}"
+        print(error_msg)
+
+        current_progress.update(
+            {
+                "status": "error",
+                "message": error_msg,
+                "progress": 0,
+                "phase": "Failed",
+                "details": str(e),
+                "error": str(e),
+                "step": "file_loading",
+            }
+        )
+
+        return current_progress
+    
+    
+def start_vector_ingestion(current_progress: Dict):
+    """Step 2: Ingest loaded documents into vector store"""
+    print("\n🔄 STARTING VECTOR INGESTION STEP")
+
+    # Check if we have loaded documents from previous step
+    if current_progress.get("step") != "file_loading_complete":
+        return {
+            "status": "error",
+            "message": "❌ No loaded documents found. Please load files first.",
+            "progress": 0,
+            "details": "",
+            "step": "vector_ingestion",
+        }
+
+    all_documents = current_progress.get("loaded_documents", [])
+    repo_name = current_progress.get("repo_name", "")
+
+    if not all_documents:
+        return {
+            "status": "error",
+            "message": "❌ No documents available for vector ingestion",
+            "progress": 0,
+            "details": "",
+            "step": "vector_ingestion",
+        }
+
+    vector_start_time = time.time()
+
+    # Update state for vector store phase
+    current_progress.update(
+        {
+            "status": "vectorizing",
+            "message": "🔄 Generating embeddings and storing in vector database",
+            "progress": 0,
+            "phase": "Vector Store Ingestion",
+            "details": f"Processing {len(all_documents)} documents for embedding...",
+            "step": "vector_ingestion",
+        }
+    )
+
+    try:
+        print("🔄 STARTING VECTOR STORE INGESTION")
+        print(f"   Repository: {repo_name}")
+        print(f"   Documents to process: {len(all_documents)}")
+
+        # Call the async ingestion function with repo name
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            loop.run_until_complete(ingest_documents_async(all_documents, repo_name))
+        finally:
+            loop.close()
+
+        vector_time = time.time() - vector_start_time
+        loading_time = current_progress.get("loading_time", 0)
+        total_time = loading_time + vector_time
+
+        print(f"✅ Vector ingestion completed in {vector_time:.2f} seconds")
+
+        failed_files_data = current_progress.get("failed_files", [])
+        if isinstance(failed_files_data, list):
+            failed_files_count = len(failed_files_data)
+        else:
+            failed_files_count = failed_files_data if isinstance(failed_files_data, int) else 0
+
+        # Update final success state with repository update flag
+        current_progress.update(
+            {
+                "status": "complete",
+                "message": "✅ Complete Ingestion Pipeline Finished!",
+                "progress": 100,
+                "phase": "Complete",
+                "details": f"Successfully processed {len(all_documents)} documents for {repo_name}",
+                "step": "complete",
+                "total_time": total_time,
+                "documents_processed": len(all_documents),
+                "failed_files_count": failed_files_count,  # Use count instead of trying len()
+                "failed_files": failed_files_data,  # Keep original data
+                "vector_time": vector_time,
+                "loading_time": loading_time,
+                "repo_name": repo_name,
+                "repository_updated": True,  # Flag to trigger repo list refresh
+            }
+        )
+
+        return current_progress
+
+    except Exception as ingest_error:
+        vector_time = time.time() - vector_start_time
+        print(f"❌ Vector ingestion failed after {vector_time:.2f} seconds")
+        print(f"❌ Error: {ingest_error}")
+
+        # Get failed files data safely
+        failed_files_data = current_progress.get("failed_files", [])
+        if isinstance(failed_files_data, list):
+            failed_files_count = len(failed_files_data)
+        else:
+            failed_files_count = failed_files_data if isinstance(failed_files_data, int) else 0
+
+        current_progress.update(
+            {
+                "status": "error",
+                "message": "❌ Vector Store Ingestion Failed",
+                "progress": 0,
+                "phase": "Failed",
+                "details": f"Error: {str(ingest_error)}",
+                "error": str(ingest_error),
+                "step": "vector_ingestion",
+                "failed_files_count": failed_files_count,
+                "failed_files": failed_files_data,
+            }
+        )
+
+        return current_progress
+
+def start_file_loading_generator(
+    repo_url: str, selected_files: List[str], current_progress: Dict
+):
+    """Step 1: Load files from GitHub with yield-based real-time updates"""
+    
+    print("\n🔄 STARTING FILE LOADING STEP")
+    print(f"📍 Repository: {repo_url}")
+    print(f"📋 Selected files: {len(selected_files)} files")
+
+    if not selected_files:
+        error_progress = {
+            "status": "error",
+            "message": "❌ No files selected for loading",
+            "progress": 0,
+            "details": "Please select at least one file to proceed.",
+            "step": "file_loading",
+        }
+        yield error_progress
+        return error_progress
+
+    total_files = len(selected_files)
+    start_time = time.time()
+
+    # Parse repo name from URL
+    if "github.com" in repo_url:
+        repo_name = (
+            repo_url.replace("https://github.com/", "")
+            .replace("http://github.com/", "")
+            .strip("/")
+        )
+        if "/" not in repo_name:
+            error_progress = {
+                "status": "error",
+                "message": "❌ Invalid repository URL format",
+                "progress": 0,
+                "details": "Expected format: owner/repo or https://github.com/owner/repo",
+                "step": "file_loading",
+            }
+            yield error_progress
+            return error_progress
+    else:
+        repo_name = repo_url.strip()
+
+    try:
+        batch_size = 10
+        all_documents = []
+        all_failed = []
+
+        # Initial progress update
+        initial_progress = {
+            "status": "loading",
+            "message": f"🚀 Starting file loading from {repo_name}",
+            "progress": 0,
+            "total_files": total_files,
+            "processed_files": 0,
+            "successful_files": 0,
+            "failed_files": 0,
+            "phase": "File Loading",
+            "details": f"Preparing to load {total_files} files in batches of {batch_size}...",
+            "step": "file_loading",
+            "current_batch": 0,
+            "total_batches": (len(selected_files) + batch_size - 1) // batch_size,
+            "repo_name": repo_name,
+        }
+        yield initial_progress
+        
+        time.sleep(0.5)
+
+        for i in range(0, len(selected_files), batch_size):
+            batch = selected_files[i : i + batch_size]
+            current_batch_num = i // batch_size + 1
+            total_batches = (len(selected_files) + batch_size - 1) // batch_size
+
+            # Update progress at batch start
+            batch_start_progress = {
+                "status": "loading",
+                "message": f"🔄 Loading batch {current_batch_num}/{total_batches}",
+                "progress": (i / total_files) * 90,
+                "processed_files": i,
+                "successful_files": len(all_documents),
+                "failed_files": len(all_failed),
+                "current_batch": current_batch_num,
+                "total_batches": total_batches,
+                "phase": "File Loading",
+                "details": f"Processing batch {current_batch_num}: {', '.join([f.split('/')[-1] for f in batch[:3]])}{'...' if len(batch) > 3 else ''}",
+                "step": "file_loading",
+                "repo_name": repo_name,
+            }
+            yield batch_start_progress
+
+            try:
+                print(f"\n📦 PROCESSING BATCH {current_batch_num}/{total_batches}")
+                print(f"   Files: {[f.split('/')[-1] for f in batch]}")
+
+                documents, failed = load_github_files(
+                    repo_name=repo_name,
+                    file_paths=batch,
+                    branch="main",
+                    concurrent_requests=10,
+                    github_token=os.getenv("GITHUB_API_KEY"),
+                )
+
+                print("✅ Load results:")
+                print(f"   - Documents: {len(documents)}")
+                print(f"   - Failed: {len(failed)}")
+
+                # Process documents
+                for j, doc in enumerate(documents):
+                    print(f"   📄 Doc {j + 1}: {doc.doc_id}")
+                    print(f"      Size: {len(doc.text)} chars")
+
+                    if "repo" not in doc.metadata:
+                        doc.metadata["repo"] = repo_name
+                        print(f"      ✅ Added repo metadata: {repo_name}")
+
+                all_documents.extend(documents)
+                all_failed.extend(failed)
+
+                # Update progress after batch completion
+                batch_complete_progress = {
+                    "status": "loading",
+                    "message": f"✅ Completed batch {current_batch_num}/{total_batches}",
+                    "progress": ((i + len(batch)) / total_files) * 90,
+                    "processed_files": i + len(batch),
+                    "successful_files": len(all_documents),
+                    "failed_files": len(all_failed),
+                    "current_batch": current_batch_num,
+                    "total_batches": total_batches,
+                    "phase": "File Loading",
+                    "details": f"✅ Batch {current_batch_num} complete: {len(documents)} loaded, {len(failed)} failed. Total progress: {len(all_documents)} documents loaded.",
+                    "step": "file_loading",
+                    "repo_name": repo_name,
+                }
+                yield batch_complete_progress
+                
+                time.sleep(0.3)
+
+            except Exception as batch_error:
+                print(f"❌ Batch processing error: {batch_error}")
+                all_failed.extend(batch)
+                
+                error_progress = {
+                    "status": "loading",
+                    "message": f"⚠️ Error in batch {current_batch_num}",
+                    "progress": ((i + len(batch)) / total_files) * 90,
+                    "processed_files": i + len(batch),
+                    "successful_files": len(all_documents),
+                    "failed_files": len(all_failed),
+                    "current_batch": current_batch_num,
+                    "phase": "File Loading",
+                    "details": f"❌ Batch {current_batch_num} error: {str(batch_error)[:100]}... Continuing with next batch.",
+                    "step": "file_loading",
+                    "repo_name": repo_name,
+                }
+                yield error_progress
+
+        loading_time = time.time() - start_time
+
+        # Final completion update
+        completion_progress = {
+            "status": "loaded",
+            "message": f"✅ File Loading Complete! Loaded {len(all_documents)} documents",
+            "progress": 100,
+            "phase": "Files Loaded Successfully",
+            "details": f"🎯 Final Results:\n✅ Successfully loaded: {len(all_documents)} documents\n❌ Failed files: {len(all_failed)}\n⏱️ Total time: {loading_time:.1f}s\n📊 Success rate: {(len(all_documents)/(len(all_documents)+len(all_failed))*100):.1f}%",
+            "step": "file_loading_complete",
+            "loaded_documents": all_documents,
+            "failed_files": all_failed,
+            "loading_time": loading_time,
+            "repo_name": repo_name,
+            "total_files": total_files,
+            "processed_files": total_files,
+            "successful_files": len(all_documents),
+        }
+        yield completion_progress
+        return completion_progress
+
+    except Exception as e:
+        total_time = time.time() - start_time
+        error_msg = f"❌ File loading error after {total_time:.1f}s: {str(e)}"
+        print(error_msg)
+
+        error_progress = {
+            "status": "error",
+            "message": error_msg,
+            "progress": 0,
+            "phase": "Loading Failed",
+            "details": f"Critical error during file loading:\n{str(e)}",
+            "error": str(e),
+            "step": "file_loading",
+        }
+        yield error_progress
+        return error_progress
+
+# Progress display component
+def format_progress_display(progress_state: Dict) -> str:
+    """Format progress state into readable display with enhanced details"""
+    if not progress_state:
+        return "🚀 Ready to start ingestion...\n\n📋 **Two-Step Process:**\n1️⃣ Load files from GitHub repository\n2️⃣ Generate embeddings and store in vector database"
+
+    status = progress_state.get("status", "unknown")
+    message = progress_state.get("message", "")
+    progress = progress_state.get("progress", 0)
+    phase = progress_state.get("phase", "")
+    details = progress_state.get("details", "")
+
+    # Enhanced progress bar
+    filled = int(progress / 2.5)  # 40 chars total
+    progress_bar = "█" * filled + "░" * (40 - filled)
+    
+    # Status emoji mapping
+    status_emoji = {
+        "loading": "⏳",
+        "loaded": "✅", 
+        "vectorizing": "🧠",
+        "complete": "🎉",
+        "error": "❌"
+    }
+    
+    emoji = status_emoji.get(status, "🔄")
+
+    output = f"{emoji} **{message}**\n\n"
+    
+    # Phase and progress section
+    output += f"📊 **Current Phase:** {phase}\n"
+    output += f"📈 **Progress:** {progress:.1f}%\n"
+    output += f"[{progress_bar}] {progress:.1f}%\n\n"
+
+    # Step-specific details for file loading
+    if progress_state.get("step") == "file_loading":
+        processed = progress_state.get("processed_files", 0)
+        total = progress_state.get("total_files", 0)
+        successful = progress_state.get("successful_files", 0)
+        failed = progress_state.get("failed_files", 0)
+        
+        if total > 0:
+            output += "📁 **File Processing Status:**\n"
+            output += f"   • Total files: {total}\n"
+            output += f"   • Processed: {processed}/{total}\n"
+            output += f"   • ✅ Successful: {successful}\n"
+            output += f"   • ❌ Failed: {failed}\n"
+            
+            if "current_batch" in progress_state and "total_batches" in progress_state:
+                output += f"   • 📦 Current batch: {progress_state['current_batch']}/{progress_state['total_batches']}\n"
+            output += "\n"
+
+    # Step-specific details for vector ingestion
+    elif progress_state.get("step") == "vector_ingestion":
+        docs_count = progress_state.get("documents_count", 0)
+        repo_name = progress_state.get("repo_name", "Unknown")
+        
+        if docs_count > 0:
+            output += "🧠 **Vector Processing Status:**\n"
+            output += f"   • Repository: {repo_name}\n"
+            output += f"   • Documents: {docs_count:,}\n"
+            output += f"   • Stage: {phase}\n\n"
+
+    # Detailed information
+    output += f"📝 **Details:**\n{details}\n"
+
+    # Final summary for completion
+    if status == "complete":
+        total_time = progress_state.get("total_time", 0)
+        docs_processed = progress_state.get("documents_processed", 0)
+        failed_files = progress_state.get("failed_files", 0)
+        vector_time = progress_state.get("vector_time", 0)
+        loading_time = progress_state.get("loading_time", 0)
+        repo_name = progress_state.get("repo_name", "Unknown")
+
+        output += "\n🎊 **INGESTION COMPLETED SUCCESSFULLY!**\n"
+        output += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
+        output += f"🎯 **Repository:** {repo_name}\n"
+        output += f"📄 **Documents processed:** {docs_processed:,}\n"
+        output += f"❌ **Failed files:** {len(failed_files) if isinstance(failed_files, list) else failed_files}\n"
+        output += f"⏱️ **Total time:** {total_time:.1f} seconds\n"
+        output += f"   ├─ File loading: {loading_time:.1f}s\n"
+        output += f"   └─ Vector processing: {vector_time:.1f}s\n"
+        output += f"📊 **Processing rate:** {docs_processed/total_time:.1f} docs/second\n\n"
+        output += "🚀 **Next Step:** Go to the 'Query Interface' tab to start asking questions!"
+
+    elif status == "error":
+        error = progress_state.get("error", "Unknown error")
+        output += "\n💥 **ERROR OCCURRED**\n"
+        output += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
+        output += f"❌ **Error Details:** {error[:300]}{'...' if len(error) > 300 else ''}\n"
+        output += "\n🔧 **Troubleshooting Tips:**\n"
+        output += "   • Check your GitHub token permissions\n"
+        output += "   • Verify repository URL format\n"
+        output += "   • Ensure selected files exist\n"
+        output += "   • Check network connectivity\n"
+
+    return output
+
+
+# Create the main Gradio interface
+with gr.Blocks(title="Doc-MCP") as demo:
+    gr.Markdown("# 📚Doc-MCP: Documentation RAG System")
+    gr.Markdown(
+        "Transform GitHub documentation repositories into accessible MCP (Model Context Protocol) servers for AI agents. Upload documentation, generate vector embeddings, and query with intelligent context retrieval."
+    )
+
+    # State variables
+    files_state = gr.State([])
+    progress_state = gr.State({})
+
+    with gr.Tabs():
+        with gr.TabItem("📥 Documentation Ingestion"):
+            gr.Markdown("### 🚀 Two-Step Documentation Processing Pipeline")
+            gr.Markdown(
+                 "**Step 1:** Fetch markdown files from GitHub repository → **Step 2:** Generate vector embeddings and store in MongoDB Atlas"
+            )
+
+            with gr.Row():
+                with gr.Column(scale=2):
+                    repo_input = gr.Textbox(
+                        label="📂 GitHub Repository URL",
+                        placeholder="Enter: owner/repo or https://github.com/owner/repo (e.g., gradio-app/gradio)",
+                        value="",
+                        info="Enter any GitHub repository containing markdown documentation"
+                    )
+                    load_btn = gr.Button("🔍 Discover Documentation Files", variant="secondary")
+
+                with gr.Column(scale=1):
+                    status_output = gr.Textbox(
+                        label="Repository Discovery Status", interactive=False, lines=4,
+                        placeholder="Repository scanning results will appear here..."
+                    )
+            with gr.Row():
+                select_all_btn = gr.Button("📋 Select All Documents", variant="secondary")
+                clear_all_btn = gr.Button("🗑️ Clear Selection", variant="secondary")
+
+            # File selection
+            with gr.Accordion(label="Available Documentation Files"):
+                file_selector = gr.CheckboxGroup(
+                    choices=[], label="Select Markdown Files for RAG Processing", visible=False
+                )
+
+            # Two-step ingestion controls
+            gr.Markdown("### 🔄 RAG Pipeline Execution")
+            gr.Markdown("Process your documentation through our advanced RAG pipeline using Nebius AI embeddings and MongoDB Atlas vector storage.")
+
+            with gr.Row():
+                with gr.Column():
+                    step1_btn = gr.Button(
+                        "📥 Step 1: Load Files from GitHub",
+                        variant="primary",
+                        size="lg",
+                        interactive=False,
+                    )
+
+                with gr.Column():
+                    step2_btn = gr.Button(
+                        "🔄 Step 2: Start Ingestion",
+                        variant="primary",
+                        size="lg",
+                        interactive=False,
+                    )
+
+            with gr.Row():
+                refresh_btn = gr.Button("🔄 Refresh Progress", variant="secondary")
+                reset_btn = gr.Button("🗑️ Reset Progress", variant="secondary")
+
+            # Progress display
+            progress_display = gr.Textbox(
+                label="📊 Real-time Ingestion Progress",
+                interactive=False,
+                lines=25,
+                value="🚀 Ready to start two-step ingestion process...\n\n📋 Steps:\n1️⃣ Load files from GitHub repository\n2️⃣ Generate embeddings and store in vector database",
+                max_lines=30,
+                show_copy_button=True,
+            )
+
+            # Event handlers
+            def load_files_handler(repo_url: str):
+                if not repo_url.strip():
+                    return (
+                        gr.CheckboxGroup(choices=[], visible=False),
+                        "Please enter a repository URL",
+                        [],
+                        gr.Button(interactive=False),
+                        gr.Button(interactive=False),
+                    )
+
+                files, message = fetch_files_with_loader(repo_url)
+
+                if files:
+                    return (
+                        gr.CheckboxGroup(
+                            choices=files,
+                            value=[],
+                            label=f"Select Files from {repo_url} ({len(files)} files)",
+                            visible=True,
+                        ),
+                        message,
+                        files,
+                        gr.Button(interactive=True),  # Enable step 1 button
+                        gr.Button(interactive=False),  # Keep step 2 disabled
+                    )
+                else:
+                    return (
+                        gr.CheckboxGroup(choices=[], visible=False),
+                        message,
+                        [],
+                        gr.Button(interactive=False),
+                        gr.Button(interactive=False),
+                    )
+
+            def start_step1_generator(repo_url: str, selected_files: List[str], current_progress: Dict):
+                """Start Step 1 with generator-based real-time progress updates"""
+                for progress_update in start_file_loading_generator(repo_url, selected_files, current_progress.copy()):
+                    progress_text = format_progress_display(progress_update)
+                    step2_enabled = progress_update.get("step") == "file_loading_complete"
+                    
+                    yield (
+                        progress_update,
+                        progress_text,
+                        gr.Button(interactive=step2_enabled),
+                    )
+
+            def start_step2(current_progress: Dict):
+                """Start Step 2: Vector Ingestion"""
+                new_progress = start_vector_ingestion(current_progress.copy())
+                progress_text = format_progress_display(new_progress)
+                return new_progress, progress_text
+
+            def refresh_progress(current_progress: Dict):
+                """Refresh the progress display"""
+                progress_text = format_progress_display(current_progress)
+                return progress_text
+
+            def reset_progress():
+                """Reset all progress"""
+                return (
+                    {},
+                    "Ready to start two-step ingestion process...",
+                    gr.Button(interactive=False),
+                )
+
+            def select_all_handler(available_files):
+                if available_files:
+                    return gr.CheckboxGroup(value=available_files)
+                return gr.CheckboxGroup(value=[])
+
+            def clear_all_handler():
+                return gr.CheckboxGroup(value=[])
+
+            # Wire up events
+            load_btn.click(
+                fn=load_files_handler,
+                inputs=[repo_input],
+                outputs=[
+                    file_selector,
+                    status_output,
+                    files_state,
+                    step1_btn,
+                    step2_btn,
+                ],
+                show_api=False,
+            )
+
+            select_all_btn.click(
+                fn=select_all_handler,
+                inputs=[files_state],
+                outputs=[file_selector],
+                show_api=False,
+            )
+
+            clear_all_btn.click(
+                fn=clear_all_handler, outputs=[file_selector], show_api=False
+            )
+
+            step1_btn.click(
+                fn=start_step1_generator,
+                inputs=[repo_input, file_selector, progress_state],
+                outputs=[progress_state, progress_display, step2_btn],
+                show_api=False,
+            )
+
+            step2_btn.click(
+                fn=start_step2,
+                inputs=[progress_state],
+                outputs=[progress_state, progress_display],
+                show_api=False,
+            )
+
+            refresh_btn.click(
+                fn=refresh_progress,
+                inputs=[progress_state],
+                outputs=[progress_display],
+                show_api=False,
+            )
+
+            reset_btn.click(
+                fn=reset_progress,
+                outputs=[progress_state, progress_display, step2_btn],
+                show_api=False,
+            )
+
+        # ================================
+        # Tab 2: Query Interface
+        # ================================
+        with gr.TabItem("🤖 AI Documentation Assistant"):
+            gr.Markdown("### 💬 Intelligent Documentation Q&A")
+            gr.Markdown(
+                "Query your processed documentation using advanced semantic search. Get contextual answers with source citations powered by Nebius LLM and vector similarity search."
+            )
+
+            with gr.Row():
+                with gr.Column(scale=2):
+                    # Repository selection
+                    repo_dropdown = gr.Dropdown(
+                        choices=get_available_repositories(),
+                        label="Select Documentation Repository",
+                        value=None,
+                        interactive=True,
+                        allow_custom_value=False,
+                    )
+                    refresh_repos_btn = gr.Button(
+                        "🔄 Refresh Repositories", variant="secondary", size="sm"
+                    )
+
+                    # Query mode selection
+                    query_mode = gr.Radio(
+                        choices=["default", "text_search", "hybrid"],
+                        label="Query Mode",
+                        value="default",
+                        info="default: semantic similarity, text_search: keyword-based, hybrid: combines both",
+                    )
+
+                    # Query input
+                    query_input = gr.Textbox(
+                        label="Your Query",
+                        placeholder="Ask about the documentation...",
+                        lines=3,
+                    )
+
+                    query_btn = gr.Button("🔍 Search", variant="primary", size="lg")
+
+                    # Response display as text area
+                    response_output = gr.Textbox(
+                        label="Response",
+                        value="Your query response will appear here...",
+                        lines=10,
+                        interactive=False,
+                    )
+
+                with gr.Column(scale=2):
+                    gr.Markdown("### Source Nodes (JSON)")
+
+                    # Source nodes display as JSON
+                    sources_output = gr.JSON(
+                        label="Source Nodes",
+                        value={
+                            "message": "Source nodes will appear here after querying..."
+                        },
+                    )
+
+            def get_available_docs_repo():
+                """
+                List the available docs of repositories
+
+                Returns:
+                    List of repo names
+                """
+                try:
+                    repos = get_available_repositories()
+                    return gr.Dropdown(choices=repos, value=repos[0] if repos else None)
+                except Exception as e:
+                    print(f"Error refreshing repository list: {e}")
+                    return gr.Dropdown(choices=[], value=None)
+
+            # Simple query handler
+            def handle_query(repo: str, mode: str, query: str):
+                """
+                Handle query request - returns raw data from retriever
+                Args:
+                    repo: Selected repository
+                    mode: Query mode (default, text_search, hybrid)
+                    query: User's query
+                Returns:
+                    Raw result dict from QueryRetriever.make_query()
+                """
+                if not query.strip():
+                    return {"error": "Please enter a query."}
+
+                if not repo:
+                    return {"error": "Please select a repository."}
+
+                try:
+                    # Import QueryRetriever here to avoid circular imports
+                    from rag.query import QueryRetriever
+
+                    # Create query retriever for the selected repo
+                    retriever = QueryRetriever(repo)
+
+                    # Make the query and return raw result
+                    result = retriever.make_query(query, mode)
+                    return result
+
+                except Exception as e:
+                    print(f"Query error: {e}")
+                    import traceback
+
+                    traceback.print_exc()
+                    return {"error": f"Query failed: {str(e)}"}
+
+            def make_query(repo: str, mode: str, query: str):
+                """
+                Retrieve relevant documentation context for a given query using specified retrieval mode.
+
+                This function is designed to support Retrieval-Augmented Generation (RAG) by extracting
+                the most relevant context chunks from indexed documentation sources.
+                Args:
+                    repo: Selected repository
+                    mode: Query mode
+                    query: User's query
+                Returns:
+                    Tuple of (response_text, source_nodes_json)
+                """
+                # Get raw result
+                result = handle_query(repo, mode, query)
+
+                # Extract response text
+                if "error" in result:
+                    response_text = f"Error: {result['error']}"
+                    source_nodes = {"error": result["error"]}
+                else:
+                    response_text = result.get("response", "No response available")
+                    source_nodes = result.get("source_nodes", [])
+
+                return response_text, source_nodes
+
+            refresh_repos_btn.click(
+                fn=get_available_docs_repo,
+                outputs=[repo_dropdown],
+                api_name="List available docs",
+            )
+
+            # Simple event wiring - single button click
+            query_btn.click(
+                fn=make_query,
+                inputs=[repo_dropdown, query_mode, query_input],
+                outputs=[response_output, sources_output],
+                api_name="Query docs",
+            )
+
+            # Also allow Enter key to trigger query
+            query_input.submit(
+                fn=make_query,
+                inputs=[repo_dropdown, query_mode, query_input],
+                outputs=[response_output, sources_output],
+                show_api=False,
+            )
+
+        # ================================
+        # Tab 3: Repository Management
+        # ================================
+        with gr.TabItem("🗂️ Repository Management"):
+            gr.Markdown("Manage your ingested repositories - view details and delete repositories when needed.")
+            
+            with gr.Row():
+                with gr.Column(scale=1):
+                    gr.Markdown("### 📊 Repository Statistics")
+                    stats_display = gr.JSON(
+                        label="Database Statistics",
+                        value={"message": "Click refresh to load statistics..."}
+                    )
+                    refresh_stats_btn = gr.Button("🔄 Refresh Statistics", variant="secondary")
+                
+                with gr.Column(scale=2):
+                    gr.Markdown("### 📋 Repository Details")
+                    repos_table = gr.Dataframe(
+                        headers=["Repository", "Files", "Last Updated"],
+                        datatype=["str", "number", "str"],
+                        label="Ingested Repositories",
+                        interactive=False,
+                        wrap=True
+                    )
+                    refresh_repos_btn = gr.Button("🔄 Refresh Repository List", variant="secondary")
+
+            gr.Markdown("### 🗑️ Delete Repository")
+            gr.Markdown("**⚠️ Warning:** This will permanently delete all documents and metadata for the selected repository.")
+            
+            with gr.Row():
+                with gr.Column(scale=2):
+                    delete_repo_dropdown = gr.Dropdown(
+                        choices=[],
+                        label="Select Repository to Delete",
+                        value=None,
+                        interactive=True,
+                        allow_custom_value=False,
+                    )
+                    
+                    # Confirmation checkbox
+                    confirm_delete = gr.Checkbox(
+                        label="I understand this action cannot be undone",
+                        value=False
+                    )
+                    
+                    delete_btn = gr.Button(
+                        "🗑️ Delete Repository", 
+                        variant="stop", 
+                        size="lg",
+                        interactive=False
+                    )
+                
+                with gr.Column(scale=1):
+                    deletion_status = gr.Textbox(
+                        label="Deletion Status",
+                        value="Select a repository and confirm to enable deletion.",
+                        interactive=False,
+                        lines=6
+                    )
+
+            # Management functions
+            def load_repository_stats():
+                """Load overall repository statistics"""
+                try:
+                    from rag.config import get_repository_stats
+                    stats = get_repository_stats()
+                    return stats
+                except Exception as e:
+                    return {"error": f"Failed to load statistics: {str(e)}"}
+
+            def load_repository_details():
+                """Load detailed repository information as a table"""
+                try:
+                    from rag.config import get_repo_details
+                    details = get_repo_details()
+                    
+                    if not details:
+                        return [["No repositories found", 0, "N/A"]]
+                    
+                    # Format for dataframe
+                    table_data = []
+                    for repo in details:
+                        last_updated = repo.get("last_updated", "Unknown")
+                        if hasattr(last_updated, 'strftime'):
+                            last_updated = last_updated.strftime("%Y-%m-%d %H:%M")
+                        elif last_updated != "Unknown":
+                            last_updated = str(last_updated)
+                        
+                        table_data.append([
+                            repo.get("repo_name", "Unknown"),
+                            repo.get("file_count", 0),
+                            last_updated
+                        ])
+                    
+                    return table_data
+                    
+                except Exception as e:
+                    return [["Error loading repositories", 0, str(e)]]
+
+            def update_delete_dropdown():
+                """Update the dropdown with available repositories"""
+                try:
+                    repos = get_available_repositories()
+                    return gr.Dropdown(choices=repos, value=None)
+                except Exception as e:
+                    print(f"Error updating delete dropdown: {e}")
+                    return gr.Dropdown(choices=[], value=None)
+
+            def check_delete_button_state(repo_selected, confirmation_checked):
+                """Enable/disable delete button based on selection and confirmation"""
+                if repo_selected and confirmation_checked:
+                    return gr.Button(interactive=True)
+                else:
+                    return gr.Button(interactive=False)
+
+            def delete_repository(repo_name: str, confirmed: bool):
+                """Delete the selected repository"""
+                if not repo_name:
+                    return "❌ No repository selected.", gr.Dropdown(choices=[]), gr.Checkbox(value=False)
+                
+                if not confirmed:
+                    return "❌ Please confirm deletion by checking the checkbox.", gr.Dropdown(choices=[]), gr.Checkbox(value=False)
+                
+                try:
+                    from rag.config import delete_repository_data
+
+                    # Perform deletion
+                    result = delete_repository_data(repo_name)
+                    
+                    # Prepare status message
+                    status_msg = result["message"]
+                    if result["success"]:
+                        status_msg += "\n\n📊 Deletion Summary:"
+                        status_msg += f"\n- Vector documents removed: {result['vector_docs_deleted']}"
+                        status_msg += f"\n- Repository record deleted: {'Yes' if result['repo_record_deleted'] else 'No'}"
+                        status_msg += f"\n\n✅ Repository '{repo_name}' has been completely removed."
+                    
+                    # Update dropdown (remove deleted repo)
+                    updated_dropdown = update_delete_dropdown()
+                    
+                    # Reset confirmation checkbox
+                    reset_checkbox = gr.Checkbox(value=False)
+                    
+                    return status_msg, updated_dropdown, reset_checkbox
+                    
+                except Exception as e:
+                    error_msg = f"❌ Error deleting repository: {str(e)}"
+                    return error_msg, gr.Dropdown(choices=[]), gr.Checkbox(value=False)
+
+            # Wire up management events
+            refresh_stats_btn.click(
+                fn=load_repository_stats,
+                outputs=[stats_display],
+                show_api=False
+            )
+            
+            refresh_repos_btn.click(
+                fn=load_repository_details,
+                outputs=[repos_table],
+                show_api=False
+            )
+            
+            # Update delete dropdown when refreshing repos
+            refresh_repos_btn.click(
+                fn=update_delete_dropdown,
+                outputs=[delete_repo_dropdown],
+                show_api=False
+            )
+            
+            # Enable/disable delete button based on selection and confirmation
+            delete_repo_dropdown.change(
+                fn=check_delete_button_state,
+                inputs=[delete_repo_dropdown, confirm_delete],
+                outputs=[delete_btn],
+                show_api=False
+            )
+            
+            confirm_delete.change(
+                fn=check_delete_button_state,
+                inputs=[delete_repo_dropdown, confirm_delete],
+                outputs=[delete_btn],
+                show_api=False
+            )
+            
+            # Delete repository
+            delete_btn.click(
+                fn=delete_repository,
+                inputs=[delete_repo_dropdown, confirm_delete],
+                outputs=[deletion_status, delete_repo_dropdown, confirm_delete],
+                show_api=False
+            )
+
+            # Load data on tab load
+            demo.load(
+                fn=load_repository_stats,
+                outputs=[stats_display],
+                show_api=False
+            )
+            
+            demo.load(
+                fn=load_repository_details,
+                outputs=[repos_table],
+                show_api=False
+            )
+            
+            demo.load(
+                fn=update_delete_dropdown,
+                outputs=[delete_repo_dropdown],
+                show_api=False
+            )
+
+
+if __name__ == "__main__":
+    demo.launch(mcp_server=True)

rag/config.py

@@ -0,0 +1,211 @@
+import os
+from typing import List
+
+from llama_index.embeddings.nebius import NebiusEmbedding
+from llama_index.llms.nebius import NebiusLLM
+from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch
+from pymongo import MongoClient
+from pymongo.operations import SearchIndexModel
+
+llm = NebiusLLM(
+    model="meta-llama/Llama-3.3-70B-Instruct-fast", api_key=os.getenv("NEBIUS_API_KEY")
+)
+embed_model = NebiusEmbedding(
+    model_name="BAAI/bge-en-icl",
+    api_key=os.getenv("NEBIUS_API_KEY"),
+    embed_batch_size=10,
+)
+
+MONGO_DB_URI = os.getenv("MONGO_DB_URI")
+mongo_client = MongoClient(MONGO_DB_URI)
+
+# Database and collection names
+DB_NAME = "docmcp"
+COLLECTION_NAME = "doc_rag"
+REPOS_COLLECTION_NAME = "ingested_repos"
+
+VS_INDEX_NAME = "vector_index"
+FTS_INDEX_NAME = "fts_index"
+vs_model = SearchIndexModel(
+    definition={
+        "fields": [
+            {
+                "type": "vector",
+                "path": "embedding",
+                "numDimensions": 4096,
+                "similarity": "cosine",
+            },
+            {"type": "filter", "path": "metadata.repo"},
+        ]
+    },
+    name=VS_INDEX_NAME,
+    type="vectorSearch",
+)
+
+fts_model = SearchIndexModel(
+    definition={"mappings": {"dynamic": False, "fields": {"text": {"type": "string"}}}},
+    name=FTS_INDEX_NAME,
+    type="search",
+)
+
+
+def get_vector_store():
+    collection = mongo_client[DB_NAME][COLLECTION_NAME]
+
+    vector_store = MongoDBAtlasVectorSearch(
+        mongodb_client=mongo_client,
+        db_name=DB_NAME,
+        collection_name=COLLECTION_NAME,
+        vector_index_name=VS_INDEX_NAME,
+        fulltext_index_name=FTS_INDEX_NAME,
+        embedding_key="embedding",
+        text_key="text",
+    )
+
+    collection.create_search_indexes(models=[vs_model, fts_model])
+
+    return vector_store
+
+
+def get_repos_collection():
+    return mongo_client[DB_NAME][REPOS_COLLECTION_NAME]
+
+
+def store_ingested_repo(repo_name: str, ingested_files: List[str]) -> bool:
+    try:
+        repos_collection = get_repos_collection()
+
+        # Simple document format
+        repo_doc = {
+            "_id": repo_name,  # Use repo name as unique ID
+            "repo_name": repo_name,
+            "ingested_files": ingested_files,
+            "file_count": len(ingested_files),
+        }
+
+        # Upsert the document (update if exists, insert if not)
+        repos_collection.replace_one({"_id": repo_name}, repo_doc, upsert=True)
+
+        print(f"✅ Stored repository: {repo_name} with {len(ingested_files)} files")
+        return True
+
+    except Exception as e:
+        print(f"❌ Error storing repository data: {e}")
+        return False
+
+
+def get_available_repos():
+    try:
+        repos_collection = get_repos_collection()
+
+        # Get all repository names
+        repos = repos_collection.find({}, {"repo_name": 1})
+        repo_list = [repo["repo_name"] for repo in repos]
+
+        if repo_list:
+            return sorted(repo_list)
+        else:
+            # Fallback to hardcoded list if no repos in database
+            return []
+
+    except Exception as e:
+        print(f"Error getting repos from database: {e}")
+        # Fallback to hardcoded list
+        return []
+
+
+def get_repo_details():
+    """Get detailed information about all repositories"""
+    try:
+        repos_collection = get_repos_collection()
+        
+        # Get all repository details
+        repos = repos_collection.find({})
+        repo_details = []
+        
+        for repo in repos:
+            repo_info = {
+                "repo_name": repo.get("repo_name", "Unknown"),
+                "file_count": repo.get("file_count", 0),
+                "last_updated": repo.get("last_updated", "Unknown"),
+                "ingested_files": repo.get("ingested_files", [])
+            }
+            repo_details.append(repo_info)
+        
+        return repo_details
+        
+    except Exception as e:
+        print(f"Error getting repo details: {e}")
+        return []
+
+def delete_repository_data(repo_name):
+    try:
+        result = {
+            "success": False,
+            "message": "",
+            "vector_docs_deleted": 0,
+            "repo_record_deleted": False,
+        }
+
+        # Delete from vector store (documents with this repo metadata)
+        collection = mongo_client[DB_NAME][COLLECTION_NAME]
+        vector_delete_result = collection.delete_many({"metadata.repo": repo_name})
+        result["vector_docs_deleted"] = vector_delete_result.deleted_count
+
+        # Delete from repos tracking collection
+        repos_collection = get_repos_collection()
+        repo_delete_result = repos_collection.delete_one({"_id": repo_name})
+        result["repo_record_deleted"] = repo_delete_result.deleted_count > 0
+
+        if result["vector_docs_deleted"] > 0 or result["repo_record_deleted"]:
+            result["success"] = True
+            result["message"] = f"✅ Successfully deleted repository '{repo_name}'"
+            if result["vector_docs_deleted"] > 0:
+                result["message"] += (
+                    f" ({result['vector_docs_deleted']} documents removed)"
+                )
+        else:
+            result["message"] = (
+                f"⚠️ Repository '{repo_name}' not found or already deleted"
+            )
+
+        print(result["message"])
+        return result
+
+    except Exception as e:
+        error_msg = f"❌ Error deleting repository '{repo_name}': {str(e)}"
+        print(error_msg)
+        return {
+            "success": False,
+            "message": error_msg,
+            "vector_docs_deleted": 0,
+            "repo_record_deleted": False,
+        }
+
+
+def get_repository_stats():
+    try:
+        repos_collection = get_repos_collection()
+        collection = mongo_client[DB_NAME][COLLECTION_NAME]
+
+        # Count total repositories
+        total_repos = repos_collection.count_documents({})
+
+        # Count total documents in vector store
+        total_docs = collection.count_documents({})
+
+        # Get total files across all repos
+        total_files = 0
+        repos = repos_collection.find({}, {"file_count": 1})
+        for repo in repos:
+            total_files += repo.get("file_count", 0)
+
+        return {
+            "total_repositories": total_repos,
+            "total_documents": total_docs,
+            "total_files": total_files,
+        }
+
+    except Exception as e:
+        print(f"Error getting repository stats: {e}")
+        return {"total_repositories": 0, "total_documents": 0, "total_files": 0}

rag/github_file_loader.py

@@ -0,0 +1,521 @@
+import asyncio
+import base64
+import logging
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import aiohttp
+import requests
+from llama_index.core.schema import Document
+
+logger = logging.getLogger(__name__)
+
+
+class GithubFileLoader:
+    """
+    GitHub file loader that fetches specific files asynchronously.
+
+    Returns LlamaIndex Document objects for each successfully loaded file.
+    """
+
+    def __init__(
+        self,
+        github_token: Optional[str] = None,
+        concurrent_requests: int = 10,
+        timeout: int = 30,
+        retries: int = 3,
+    ):
+        """
+        Initialize GitHub file loader.
+
+        Args:
+            github_token: GitHub API token for higher rate limits
+            concurrent_requests: Number of concurrent requests
+            timeout: Request timeout in seconds
+            retries: Number of retry attempts for failed requests
+        """
+        self.github_token = github_token
+        self.concurrent_requests = concurrent_requests
+        self.timeout = timeout
+        self.retries = retries
+
+        # Setup headers
+        self.headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "User-Agent": "LlamaIndex-GitHub-Loader/1.0",
+        }
+
+        if self.github_token:
+            self.headers["Authorization"] = f"token {self.github_token}"
+
+    def fetch_repository_files(
+        self,
+        repo_url: str,
+        file_extensions: List[str] = [".md", ".mdx"],
+        branch: str = "main",
+    ) -> Tuple[List[str], str]:
+        """
+        Fetch files from GitHub repository using GitHub API
+
+        Args:
+            repo_url: GitHub repository URL or owner/repo format
+            file_extensions: List of file extensions to filter (e.g., [".md", ".mdx", ".txt"])
+            branch: Branch name to fetch from
+
+        Returns:
+            Tuple of (list_of_file_paths, status_message)
+        """
+        try:
+            # Parse GitHub URL to extract owner and repo
+            repo_name = self._parse_repo_name(repo_url)
+            if not repo_name:
+                return (
+                    [],
+                    "Invalid GitHub URL format. Use: https://github.com/owner/repo or owner/repo",
+                )
+
+            # GitHub API endpoint for repository tree
+            api_url = f"https://api.github.com/repos/{repo_name}/git/trees/{branch}?recursive=1"
+
+            # Make request with authentication if token is available
+            response = requests.get(api_url, headers=self.headers, timeout=self.timeout)
+
+            if response.status_code == 200:
+                data = response.json()
+                filtered_files = []
+
+                # Filter for specified file extensions
+                for item in data.get("tree", []):
+                    if item["type"] == "blob":
+                        file_path = item["path"]
+                        # Check if file has any of the specified extensions
+                        if any(
+                            file_path.lower().endswith(ext.lower())
+                            for ext in file_extensions
+                        ):
+                            filtered_files.append(file_path)
+
+                if filtered_files:
+                    ext_str = ", ".join(file_extensions)
+                    return (
+                        filtered_files,
+                        f"Found {len(filtered_files)} files with extensions ({ext_str}) in {repo_name}/{branch}",
+                    )
+                else:
+                    ext_str = ", ".join(file_extensions)
+                    return (
+                        [],
+                        f"No files with extensions ({ext_str}) found in repository {repo_name}/{branch}",
+                    )
+
+            elif response.status_code == 404:
+                return (
+                    [],
+                    f"Repository '{repo_name}' not found or branch '{branch}' doesn't exist",
+                )
+            elif response.status_code == 403:
+                if "rate limit" in response.text.lower():
+                    return (
+                        [],
+                        "GitHub API rate limit exceeded. Consider using a GitHub token.",
+                    )
+                else:
+                    return (
+                        [],
+                        "Access denied. Repository may be private or require authentication.",
+                    )
+            else:
+                return (
+                    [],
+                    f"GitHub API Error: {response.status_code} - {response.text[:200]}",
+                )
+
+        except requests.exceptions.Timeout:
+            return [], f"Request timeout after {self.timeout} seconds"
+        except requests.exceptions.RequestException as e:
+            return [], f"Network error: {str(e)}"
+        except Exception as e:
+            return [], f"Unexpected error: {str(e)}"
+
+    def _parse_repo_name(self, repo_url: str) -> Optional[str]:
+        """
+        Parse repository URL to extract owner/repo format
+
+        Args:
+            repo_url: GitHub repository URL or owner/repo format
+
+        Returns:
+            Repository name in "owner/repo" format or None if invalid
+        """
+        if "github.com" in repo_url:
+            # Extract from full URL
+            parts = (
+                repo_url.replace("https://github.com/", "")
+                .replace("http://github.com/", "")
+                .strip("/")
+                .split("/")
+            )
+            if len(parts) >= 2:
+                return f"{parts[0]}/{parts[1]}"
+        else:
+            # Assume format is owner/repo
+            parts = repo_url.strip().split("/")
+            if len(parts) == 2 and all(part.strip() for part in parts):
+                return repo_url.strip()
+
+        return None
+
+    def fetch_markdown_files(
+        self, repo_url: str, branch: str = "main"
+    ) -> Tuple[List[str], str]:
+        """
+        Fetch markdown files from GitHub repository (backward compatibility method)
+
+        Args:
+            repo_url: GitHub repository URL or owner/repo format
+            branch: Branch name to fetch from
+
+        Returns:
+            Tuple of (list_of_markdown_files, status_message)
+        """
+        return self.fetch_repository_files(
+            repo_url=repo_url, file_extensions=[".md", ".mdx"], branch=branch
+        )
+
+    async def load_files(
+        self, repo_name: str, file_paths: List[str], branch: str = "main"
+    ) -> Tuple[List[Document], List[str]]:
+        """
+        Load files from GitHub repository asynchronously.
+
+        Args:
+            repo_name: Repository name in format "owner/repo"
+            file_paths: List of file paths to load
+            branch: Branch name to load from
+
+        Returns:
+            Tuple of (successfully_loaded_documents, failed_file_paths)
+        """
+        if not file_paths:
+            return [], []
+
+        # Validate repo name format
+        if not re.match(r"^[^/]+/[^/]+$", repo_name):
+            raise ValueError(f"Invalid repo format: {repo_name}. Expected 'owner/repo'")
+
+        # Create semaphore to limit concurrent requests
+        semaphore = asyncio.Semaphore(self.concurrent_requests)
+
+        # Create session
+        connector = aiohttp.TCPConnector(limit=self.concurrent_requests)
+        timeout_config = aiohttp.ClientTimeout(total=self.timeout)
+
+        async with aiohttp.ClientSession(
+            headers=self.headers, connector=connector, timeout=timeout_config
+        ) as session:
+            # Create tasks for all files
+            tasks = []
+            for file_path in file_paths:
+                task = asyncio.create_task(
+                    self._fetch_file_with_retry(
+                        session, semaphore, repo_name, file_path, branch
+                    )
+                )
+                tasks.append(task)
+
+            # Wait for all tasks to complete
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Process results
+        documents = []
+        failed_files = []
+
+        for i, result in enumerate(results):
+            file_path = file_paths[i]
+
+            if isinstance(result, Exception):
+                logger.error(f"Failed to load {file_path}: {result}")
+                failed_files.append(file_path)
+            elif result is None:
+                logger.warning(f"No content returned for {file_path}")
+                failed_files.append(file_path)
+            else:
+                documents.append(result)
+
+        logger.info(
+            f"Successfully loaded {len(documents)} files, failed: {len(failed_files)}"
+        )
+        return documents, failed_files
+
+    async def _fetch_file_with_retry(
+        self,
+        session: aiohttp.ClientSession,
+        semaphore: asyncio.Semaphore,
+        repo_name: str,
+        file_path: str,
+        branch: str,
+    ) -> Optional[Document]:
+        """Fetch a single file with retry logic."""
+        async with semaphore:
+            for attempt in range(self.retries + 1):
+                try:
+                    return await self._fetch_single_file(
+                        session, repo_name, file_path, branch
+                    )
+                except Exception as e:
+                    if attempt == self.retries:
+                        logger.error(
+                            f"Failed to fetch {file_path} after {self.retries + 1} attempts: {e}"
+                        )
+                        raise
+                    else:
+                        logger.warning(
+                            f"Attempt {attempt + 1} failed for {file_path}: {e}"
+                        )
+                        await asyncio.sleep(2**attempt)  # Exponential backoff
+
+        return None
+
+    async def _fetch_single_file(
+        self,
+        session: aiohttp.ClientSession,
+        repo_name: str,
+        file_path: str,
+        branch: str,
+    ) -> Document:
+        """Fetch a single file from GitHub API."""
+        # Clean file path
+        clean_path = file_path.strip("/")
+
+        # Build API URL
+        api_url = f"https://api.github.com/repos/{repo_name}/contents/{clean_path}"
+        params = {"ref": branch}
+
+        logger.debug(f"Fetching: {api_url}")
+
+        async with session.get(api_url, params=params) as response:
+            if response.status == 404:
+                raise FileNotFoundError(f"File not found: {file_path}")
+            elif response.status == 403:
+                raise PermissionError("API rate limit exceeded or access denied")
+            elif response.status != 200:
+                raise Exception(f"HTTP {response.status}: {await response.text()}")
+
+            data = await response.json()
+
+            # Handle directory case
+            if isinstance(data, list):
+                raise ValueError(f"Path {file_path} is a directory, not a file")
+
+            # Decode file content
+            if data.get("encoding") == "base64":
+                try:
+                    content_bytes = base64.b64decode(data["content"])
+                    content_text = content_bytes.decode("utf-8")
+                except Exception as e:
+                    logger.warning(f"Failed to decode {file_path}: {e}")
+                    # Try to decode as latin-1 as fallback
+                    content_text = content_bytes.decode("latin-1", errors="ignore")
+            else:
+                raise ValueError(f"Unsupported encoding: {data.get('encoding')}")
+
+            # Create Document
+            document = self._create_document(
+                content=content_text,
+                file_path=clean_path,
+                repo_name=repo_name,
+                branch=branch,
+                file_data=data,
+            )
+
+            return document
+
+    def _create_document(
+        self, content: str, file_path: str, repo_name: str, branch: str, file_data: Dict
+    ) -> Document:
+        """Create a LlamaIndex Document from file content and metadata."""
+
+        # Extract file info
+        filename = Path(file_path).name
+        file_extension = Path(file_path).suffix.lower()
+        directory = (
+            str(Path(file_path).parent) if Path(file_path).parent != Path(".") else ""
+        )
+
+        # Build URLs
+        html_url = f"https://github.com/{repo_name}/blob/{branch}/{file_path}"
+        raw_url = file_data.get("download_url", "")
+
+        # Create metadata
+        metadata = {
+            "file_path": file_path,
+            "file_name": filename,
+            "file_extension": file_extension,
+            "directory": directory,
+            "repo": repo_name,
+            "branch": branch,
+            "sha": file_data.get("sha", ""),
+            "size": file_data.get("size", 0),
+            "url": html_url,
+            "raw_url": raw_url,
+            "type": file_data.get("type", "file"),
+        }
+
+        # Create document with unique ID
+        doc_id = f"{repo_name}:{branch}:{file_path}"
+
+        document = Document(
+            text=content,
+            doc_id=doc_id,
+            metadata=metadata,  # For backward compatibility
+        )
+
+        return document
+
+    def load_files_sync(
+        self, repo_name: str, file_paths: List[str], branch: str = "main"
+    ) -> Tuple[List[Document], List[str]]:
+        """
+        Synchronous wrapper for load_files.
+
+        Args:
+            repo_name: Repository name in format "owner/repo"
+            file_paths: List of file paths to load
+            branch: Branch name to load from
+
+        Returns:
+            Tuple of (successfully_loaded_documents, failed_file_paths)
+        """
+
+        return asyncio.run(self.load_files(repo_name, file_paths, branch))
+
+
+# Convenience functions
+async def load_github_files_async(
+    repo_name: str,
+    file_paths: List[str],
+    branch: str = "main",
+    github_token: Optional[str] = None,
+    concurrent_requests: int = 10,
+) -> Tuple[List[Document], List[str]]:
+    """
+    Convenience function to load GitHub files asynchronously.
+
+    Args:
+        repo_name: Repository name in format "owner/repo"
+        file_paths: List of file paths to load
+        branch: Branch name to load from
+        github_token: GitHub API token
+        concurrent_requests: Number of concurrent requests
+
+    Returns:
+        Tuple of (documents, failed_files)
+    """
+    loader = GithubFileLoader(
+        github_token=github_token, concurrent_requests=concurrent_requests
+    )
+    return await loader.load_files(repo_name, file_paths, branch)
+
+
+def load_github_files(
+    repo_name: str,
+    file_paths: List[str],
+    branch: str = "main",
+    github_token: Optional[str] = None,
+    concurrent_requests: int = 10,
+) -> Tuple[List[Document], List[str]]:
+    """
+    Convenience function to load GitHub files synchronously.
+
+    Args:
+        repo_name: Repository name in format "owner/repo"
+        file_paths: List of file paths to load
+        branch: Branch name to load from
+        github_token: GitHub API token
+        concurrent_requests: Number of concurrent requests
+
+    Returns:
+        Tuple of (documents, failed_files)
+    """
+    loader = GithubFileLoader(
+        github_token=github_token, concurrent_requests=concurrent_requests
+    )
+    return loader.load_files_sync(repo_name, file_paths, branch)
+
+
+def fetch_markdown_files(
+    repo_url: str, github_token: Optional[str] = None, branch: str = "main"
+) -> Tuple[List[str], str]:
+    """
+    Convenience function to fetch markdown files from GitHub repository
+
+    Args:
+        repo_url: GitHub repository URL or owner/repo format
+        github_token: GitHub API token for higher rate limits
+        branch: Branch name to fetch from
+
+    Returns:
+        Tuple of (list_of_files, status_message)
+    """
+    loader = GithubFileLoader(github_token=github_token)
+    return loader.fetch_markdown_files(repo_url, branch)
+
+
+def fetch_repository_files(
+    repo_url: str,
+    file_extensions: List[str] = [".md", ".mdx"],
+    github_token: Optional[str] = None,
+    branch: str = "main",
+) -> Tuple[List[str], str]:
+    """
+    Convenience function to fetch files with specific extensions from GitHub repository
+
+    Args:
+        repo_url: GitHub repository URL or owner/repo format
+        file_extensions: List of file extensions to filter
+        github_token: GitHub API token for higher rate limits
+        branch: Branch name to fetch from
+
+    Returns:
+        Tuple of (list_of_files, status_message)
+    """
+    loader = GithubFileLoader(github_token=github_token)
+    return loader.fetch_repository_files(repo_url, file_extensions, branch)
+
+
+# Example usage
+if __name__ == "__main__":
+    # Example file paths
+    file_paths = [
+        "docs/contribute/docs.mdx",
+        "docs/contribute/ml-handlers.mdx",
+        "docs/contribute/community.mdx",
+        "docs/contribute/python-coding-standards.mdx",
+        "docs/features/data-integrations.mdx",
+        "docs/features/ai-integrations.mdx",
+        "docs/integrations/ai-engines/langchain_embedding.mdx",
+        "docs/integrations/ai-engines/langchain.mdx",
+        "docs/integrations/ai-engines/google_gemini.mdx",
+        "docs/integrations/ai-engines/anomaly.mdx",
+        "docs/integrations/ai-engines/amazon-bedrock.mdx",
+    ]
+
+    # Load files synchronously
+    documents, failed = load_github_files(
+        repo_name="mindsdb/mindsdb",
+        file_paths=file_paths,
+        branch="main",  # Optional
+    )
+
+    print(f"Loaded {len(documents)} documents")
+    print(f"Failed to load {len(failed)} files: {failed}")
+
+    # Print first document info
+    if documents:
+        doc = documents[0]
+        print("\nFirst document:")
+        print(f"ID: {doc.doc_id}")
+        print(f"File: {doc.metadata['file_path']}")
+        print(f"Size: {len(doc.text)} characters")
+        print(f"Content preview: {doc.text[:200]}...")

rag/ingest.py

@@ -0,0 +1,66 @@
+from typing import List
+
+from llama_index.core import StorageContext, VectorStoreIndex
+from llama_index.core.schema import Document
+
+from .config import get_vector_store, store_ingested_repo
+
+
+async def ingest_documents_async(documents: List[Document], repo_name: str = None):
+    """Async version of document ingestion with detailed logging and repo tracking"""
+    print(f"🔄 Starting async ingestion of {len(documents)} documents")
+    
+    if repo_name:
+        print(f"📍 Repository: {repo_name}")
+
+    try:
+        # Get vector store
+        vector_store = get_vector_store()
+        print(f"✅ Vector store retrieved: {type(vector_store)}")
+
+        # Create storage context
+        vector_store_context = StorageContext.from_defaults(vector_store=vector_store)
+        print(f"✅ Vector Store context created: {type(vector_store_context)}")
+
+        # Process documents and ensure repo metadata
+        print("🔄 Processing documents through pipeline...")
+        ingested_files = []
+        
+        for i, doc in enumerate(documents):
+            print(f"📄 Doc {i + 1}: {doc.doc_id} - {len(doc.text)} chars")
+            print(f"   Metadata: {doc.metadata}")
+            
+            # Ensure repo metadata is properly set
+            if repo_name and "repo" not in doc.metadata:
+                doc.metadata["repo"] = repo_name
+                print(f"   ✅ Added repo metadata: {repo_name}")
+            
+            # Track ingested file paths
+            file_path = doc.metadata.get("file_path", doc.doc_id)
+            if file_path not in ingested_files:
+                ingested_files.append(file_path)
+
+        # Run the ingestion
+        print("🚀 Starting vector store ingestion...")
+        vc_store_index = VectorStoreIndex.from_documents(
+            documents=documents,
+            storage_context=vector_store_context,
+            show_progress=True,
+        )
+        print("✅ Document Ingestion completed Successfully")
+
+        # Store repository metadata if repo_name is provided
+        if repo_name and ingested_files:
+            store_success = store_ingested_repo(repo_name, ingested_files)
+            if store_success:
+                print(f"✅ Repository metadata stored for {repo_name}")
+            else:
+                print(f"⚠️ Failed to store repository metadata for {repo_name}")
+
+        return vc_store_index
+
+    except Exception as e:
+        print(f"❌ Error in async ingestion: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        raise e

rag/query.py

@@ -0,0 +1,94 @@
+from typing import List, Optional
+
+from llama_index.core import VectorStoreIndex
+from llama_index.core.vector_stores import (FilterOperator, MetadataFilter,
+                                            MetadataFilters)
+from pydantic import BaseModel, Field
+
+from .config import get_vector_store
+
+
+class Node(BaseModel):
+    file_name: str = Field("Name of the file")
+    url: str = Field("GitHub repo url of the file")
+    score: float = Field("Relevance score of the node")  # Changed to float
+    content: str = Field("Content of the node")
+
+
+class ContextResponseModel(BaseModel):
+    response: str = Field("Response for user's query")
+    source_nodes: Optional[List[Node]] = Field(
+        "List of sources used to generate response"
+    )
+
+
+class QueryRetriever:
+    def __init__(self, repo):
+        self.vector_store_index = VectorStoreIndex.from_vector_store(get_vector_store())
+        self.filters = MetadataFilters(
+            filters=[
+                MetadataFilter(
+                    key="metadata.repo",
+                    value=repo,
+                    operator=FilterOperator.EQ,
+                )
+            ]
+        )
+
+    def make_query(self, query: str, mode: str = "default") -> dict:
+        """
+        Retrieve relevant documentation context for a given query using specified retrieval mode.
+
+        This function is designed to support Retrieval-Augmented Generation (RAG) by extracting
+        the most relevant context chunks from indexed documentation sources.
+
+        Args:
+            query (str): The user's input query related to the documentation.
+            mode (str, optional): Retrieval strategy to use. One of:
+                - "default": Standard semantic similarity search.
+                - "text_search": Keyword-based search.
+                - "hybrid": Combines semantic and keyword-based methods.
+                Defaults to "default".
+
+        Returns:
+            dict: Dictionary with 'response' and 'source_nodes' keys
+        """
+        query_engine = self.vector_store_index.as_query_engine(
+            similarity_top_k=5,
+            vector_store_query_mode=mode,
+            filters=self.filters,
+            response_mode="refine",
+        )
+
+        
+
+        response = query_engine.query(query)
+        nodes = []
+        for node in response.source_nodes:
+            nodes.append(
+                {
+                    "file_name": node.metadata.get("file_name", "Unknown"),
+                    "url": node.metadata.get("url", "#"),
+                    "score": float(node.score) if node.score else 0.0,
+                    "content": node.get_content(),
+                }
+            )
+
+        return {"response": str(response.response), "source_nodes": nodes}
+
+    @staticmethod
+    def get_available_repos() -> List[str]:
+        """Get list of available repositories in the vector store"""
+        try:
+            from .config import get_available_repos as get_repos_from_db
+            print("fetching repos")
+            re = get_repos_from_db()
+
+            print(re)
+            return re
+        except Exception as e:
+            print(f"Error getting repos from database: {e}")
+            # Fallback to hardcoded list
+            return ["mindsdb/mindsdb", "run-llama/llama_index"]
+    
+