Upload 19 files

app.py

@@ -0,0 +1,432 @@
+import streamlit as st
+import asyncio
+from src.vectorstore.pinecone_db import ingest_data, get_retriever, load_documents, process_chunks, save_to_parquet
+from src.agents.research_agent import create_industry_research_workflow
+from src.agents.workflow import run_adaptive_rag
+from pinecone import Pinecone
+from langchain_openai import ChatOpenAI
+from langchain_ollama import ChatOllama
+from langgraph.pregel import GraphRecursionError
+import tempfile
+import os
+import time
+from pathlib import Path
+
+# Page configuration
+st.set_page_config(
+    page_title="Research & RAG Assistant",
+    page_icon="🤖",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+# Custom CSS for better UI
+st.markdown("""
+    <style>
+    .stTabs [data-baseweb="tab-list"] {
+        gap: 24px;
+    }
+    .stTabs [data-baseweb="tab"] {
+        padding: 8px 16px;
+    }
+    .config-section {
+        background-color: #f0f2f6;
+        border-radius: 10px;
+        padding: 20px;
+        margin: 10px 0;
+    }
+    .chat-container {
+        border: 1px solid #e0e0e0;
+        border-radius: 10px;
+        padding: 20px;
+        margin-top: 20px;
+    }
+    .stButton>button {
+        width: 100%;
+    }
+    </style>
+""", unsafe_allow_html=True)
+
+# Initialize session states
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+if "documents_processed" not in st.session_state:
+    st.session_state.documents_processed = False
+if "retriever" not in st.session_state:
+    st.session_state.retriever = None
+if "pinecone_client" not in st.session_state:
+    st.session_state.pinecone_client = None
+if "research_config_saved" not in st.session_state:
+    st.session_state.research_config_saved = False
+if "rag_config_saved" not in st.session_state:
+    st.session_state.rag_config_saved = False
+
+def save_research_config(api_keys):
+    """Save research configuration."""
+    st.session_state.research_openai_key = api_keys['openai']
+    st.session_state.research_tavily_key = api_keys['tavily']
+    st.session_state.research_config_saved = True
+
+
+def research_config_section():
+    """Configuration section for Company Research tab."""
+    st.markdown("### ⚙️ Configuration")
+    
+    with st.expander("API Configuration", expanded=not st.session_state.research_config_saved):
+        col1, col2 = st.columns(2)
+        with col1:
+            openai_key = st.text_input(
+                "OpenAI API Key",
+                type="password",
+                value=st.session_state.get('research_openai_key', ''),
+                key="research_openai_input"
+            )
+        with col2:
+            tavily_key = st.text_input(
+                "Tavily API Key",
+                type="password",
+                value=st.session_state.get('research_tavily_key', ''),
+                key="research_tavily_input"
+            )
+        
+        if st.button("Save Research Configuration", key="save_research_config"):
+            if openai_key and tavily_key:
+                save_research_config({
+                    'openai': openai_key,
+                    'tavily': tavily_key
+                })
+                if not os.environ.get("TAVILY_API_KEY"):
+                    os.environ["TAVILY_API_KEY"] = tavily_key
+                st.success("✅ Research configuration saved!")
+            else:
+                st.error("Please provide both API keys.")
+
+
+async def run_industry_research(company: str, industry: str, llm):
+    """Run the industry research workflow asynchronously."""
+    workflow = create_industry_research_workflow(llm)
+    
+    output = await workflow.ainvoke(input={
+        "company": company,
+        "industry": industry
+    }, config={"recursion_limit": 5})
+    
+    return output['final_report']
+
+
+def research_input_section():
+    """Input section for Company Research tab."""
+    st.markdown("### 🔍 Research Parameters")
+    
+    col1, col2 = st.columns(2)
+    with col1:
+        company_name = st.text_input(
+            "Company Name",
+            placeholder="e.g., Tesla",
+            help="Enter the name of the company to research"
+        )
+    with col2:
+        industry_type = st.text_input(
+            "Industry Type",
+            placeholder="e.g., Automotive",
+            help="Enter the industry sector"
+        )
+    
+    if st.button("Generate Research Report", 
+                 disabled=not st.session_state.research_config_saved,
+                 type="primary"):
+        if company_name and industry_type:
+            with st.spinner("🔄 Generating comprehensive research report..."):
+                # try:
+                # Initialize LLM and run research
+                llm = ChatOpenAI(
+                    model="gpt-3.5-turbo-0125",
+                    temperature=0.1,
+                    api_key=st.session_state.research_openai_key
+                )
+                
+                report_path = asyncio.run(run_industry_research(
+                    company=company_name,
+                    industry=industry_type,
+                    llm=llm
+                ))
+                
+                if os.path.exists(report_path):
+                    with open(report_path, "rb") as file:
+                        st.download_button(
+                            "📥 Download Research Report",
+                            data=file,
+                            file_name=f"{company_name}_research_report.pdf",
+                            mime="application/pdf"
+                        )
+                else:
+                    st.error("Report generation failed.")
+                # except Exception as e:
+                #     st.error(f"Error during report generation: {str(e)}")
+        else:
+            st.warning("Please fill in both company name and industry type.")
+            
+def initialize_pinecone(api_key):
+    """Initialize Pinecone client with API key."""
+    try:
+        return Pinecone(api_key=api_key)
+    except Exception as e:
+        st.error(f"Error initializing Pinecone: {str(e)}")
+        return None
+
+def initialize_llm(llm_option, openai_api_key=None):
+    """Initialize LLM based on user selection."""
+    if llm_option == "OpenAI":
+        if not openai_api_key:
+            st.sidebar.warning("Please enter OpenAI API key.")
+            return None
+        return ChatOpenAI(api_key=openai_api_key, model="gpt-3.5-turbo")
+
+def clear_pinecone_index(pc, index_name="vector-index"):
+    """Clear the Pinecone index."""
+    try:
+        pc.delete_index(index_name)
+        st.session_state.documents_processed = False
+        st.session_state.retriever = None
+        st.success("Database cleared successfully!")
+    except Exception as e:
+        st.error(f"Error clearing database: {str(e)}")
+
+def process_documents(uploaded_files, pc):
+    """Process uploaded documents and store in Pinecone."""
+    if not uploaded_files:
+        st.warning("Please upload at least one document.")
+        return False
+
+    with st.spinner("Processing documents..."):
+        temp_dir = tempfile.mkdtemp()
+        file_paths = []
+        markdown_path = Path(temp_dir) / "combined.md"
+        parquet_path = Path(temp_dir) / "documents.parquet"
+        
+        for uploaded_file in uploaded_files:
+            file_path = Path(temp_dir) / uploaded_file.name
+            with open(file_path, "wb") as f:
+                f.write(uploaded_file.getvalue())
+            file_paths.append(str(file_path))
+
+        try:
+            markdown_path = load_documents(file_paths, output_path=markdown_path)
+            chunks = process_chunks(markdown_path, chunk_size=256, threshold=0.6)
+            print(f"Processed chunks: {chunks}")
+            parquet_path = save_to_parquet(chunks, parquet_path)
+            
+            ingest_data(
+                pc=pc,
+                parquet_path=parquet_path,
+                text_column="text",
+                pinecone_client=pc
+            )
+            
+            st.session_state.retriever = get_retriever(pc)
+            st.session_state.documents_processed = True
+            
+            return True
+            
+        except Exception as e:
+            st.error(f"Error processing documents: {str(e)}")
+            return False
+        finally:
+            for file_path in file_paths:
+                try:
+                    os.remove(file_path)
+                except:
+                    pass
+            try:
+                os.rmdir(temp_dir)
+            except:
+                pass
+
+def run_rag_with_streaming(retriever, question, llm, enable_web_search=False):
+    """Run RAG workflow and yield streaming results."""
+    try:
+        response = run_adaptive_rag(
+            retriever=retriever,
+            question=question,
+            llm=llm,
+            top_k=5,
+            enable_websearch=enable_web_search
+        )
+        
+        for word in response.split():
+            yield word + " "
+            time.sleep(0.03)
+            
+    except GraphRecursionError:
+        response = "I apologize, but I cannot find a sufficient answer to your question in the provided documents. Please try rephrasing your question or ask something else about the content of the documents."
+        for word in response.split():
+            yield word + " "
+            time.sleep(0.03)
+            
+    except Exception as e:
+        yield f"I encountered an error while processing your question: {str(e)}"
+
+
+def document_upload_section():
+    """Document upload section for RAG tab."""
+    st.markdown("### 📄 Document Management")
+    
+    if not st.session_state.documents_processed:
+        uploaded_files = st.file_uploader(
+            "Upload your documents",
+            accept_multiple_files=True,
+            type=["pdf", "docx", "txt", "pptx", "md"],
+            help="Support multiple file uploads"
+        )
+        
+        col1, col2 = st.columns([3, 1])
+        with col1:
+            if uploaded_files:
+                st.info(f"📁 {len(uploaded_files)} files selected")
+        with col2:
+            if st.button(
+                "Process Documents",
+                disabled=not (uploaded_files and st.session_state.rag_config_saved)
+            ):
+                if process_documents(uploaded_files, st.session_state.pinecone_client):
+                    st.success("✅ Documents processed successfully!")
+    else:
+        st.success("✅ Documents are loaded and ready for querying!")
+        if st.button("Upload New Documents"):
+            st.session_state.documents_processed = False
+            st.rerun()
+
+# Update the save_rag_config function to remove web_search
+def save_rag_config(config):
+    """Save RAG configuration."""
+    st.session_state.rag_pinecone_key = config['pinecone']
+    st.session_state.rag_openai_key = config['openai']
+    st.session_state.rag_config_saved = True
+
+# Update the rag_config_section to remove web_search checkbox
+def rag_config_section():
+    """Configuration section for RAG tab."""
+    st.markdown("### ⚙️ Configuration")
+    
+    with st.expander("API Configuration", expanded=not st.session_state.rag_config_saved):
+        col1, col2 = st.columns(2)
+        with col1:
+            pinecone_key = st.text_input(
+                "Pinecone API Key",
+                type="password",
+                value=st.session_state.get('rag_pinecone_key', ''),
+                key="rag_pinecone_input"
+            )
+        with col2:
+            openai_key = st.text_input(
+                "OpenAI API Key",
+                type="password",
+                value=st.session_state.get('rag_openai_key', ''),
+                key="rag_openai_input"
+            )
+        
+        if st.button("Save RAG Configuration", key="save_rag_config"):
+            if pinecone_key and openai_key:
+                save_rag_config({
+                    'pinecone': pinecone_key,
+                    'openai': openai_key
+                })
+                # Initialize Pinecone client
+                st.session_state.pinecone_client = initialize_pinecone(pinecone_key)
+                st.success("✅ RAG configuration saved!")
+            else:
+                st.error("Please provide both API keys.")
+
+# Update the chat_interface function to include web search toggle
+def chat_interface():
+    """Enhanced chat interface with streaming responses and web search toggle."""
+    st.markdown("### 💬 Chat Interface")
+    
+    # Add web search toggle in the chat interface
+    col1, col2 = st.columns([3, 1])
+    with col2:
+        web_search = st.checkbox(
+            "🌐 Enable Web Search",
+            value=st.session_state.get('use_web_search', False),
+            help="Toggle web search for additional context",
+            key="web_search_toggle"
+        )
+    st.session_state.use_web_search = web_search
+    
+    # Chat container with messages
+    chat_container = st.container()
+    with chat_container:
+        for message in st.session_state.messages:
+            with st.chat_message(message["role"]):
+                st.markdown(message["content"])
+    
+    # Chat input
+    if prompt := st.chat_input(
+        "Ask a question about your documents...",
+        disabled=not st.session_state.documents_processed,
+        key="chat_input"
+    ):
+        # User message
+        with st.chat_message("user"):
+            if st.session_state.use_web_search:
+                st.markdown(f"{prompt} 🌐")
+            else:
+                st.markdown(prompt)
+        st.session_state.messages.append({"role": "user", "content": prompt})
+        
+        # Assistant response
+        with st.chat_message("assistant"):
+            response_container = st.empty()
+            full_response = ""
+            
+            try:
+                with st.spinner("Thinking..."):
+                    llm = ChatOpenAI(
+                        api_key=st.session_state.rag_openai_key,
+                        model="gpt-3.5-turbo"
+                    )
+                    
+                    for chunk in run_rag_with_streaming(
+                        retriever=st.session_state.retriever,
+                        question=prompt,
+                        llm=llm,
+                        enable_web_search=st.session_state.use_web_search
+                    ):
+                        full_response += chunk
+                        response_container.markdown(full_response + "▌")
+                
+                response_container.markdown(full_response)
+                st.session_state.messages.append(
+                    {"role": "assistant", "content": full_response}
+                )
+            
+            except Exception as e:
+                st.error(f"Error: {str(e)}")
+
+def main():
+    """Main application layout."""
+    st.title("🤖 Research & RAG Assistant")
+    
+    tab1, tab2 = st.tabs(["🔍 Company Research", "💬 Document Q&A"])
+    
+    with tab1:
+        research_config_section()
+        if st.session_state.research_config_saved:
+            st.divider()
+            research_input_section()
+        else:
+            st.info("👆 Please configure your API keys above to get started.")
+    
+    with tab2:
+        rag_config_section()
+        if st.session_state.rag_config_saved:
+            st.divider()
+            document_upload_section()
+            if st.session_state.documents_processed:
+                st.divider()
+                chat_interface()
+        else:
+            st.info("👆 Please configure your API keys above to get started.")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,24 @@
+langchain-community 
+tiktoken 
+langchain-openai 
+langchainhub 
+chromadb 
+langchain 
+langgraph  
+duckduckgo-search
+langchain-groq
+langchain-huggingface
+sentence_transformers
+tavily-python
+langchain-ollama
+ollama
+crawl4ai
+docling
+easyocr
+FlagEmbedding
+chonkie[semantic]
+pinecone
+streamlit
+markdown2 
+xhtml2pdf
+PyPDF2

src/agents/research_agent.py

@@ -0,0 +1,476 @@
+from langgraph.graph import END, StateGraph, START
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain_openai import ChatOpenAI
+
+import re
+import asyncio
+from typing import TypedDict, List, Optional, Dict
+from src.tools.deep_crawler import DeepWebCrawler, ResourceCollectionAgent
+
+class ResearchGraphState(TypedDict):
+    company: str
+    industry: str
+    research_results: Optional[dict]
+    use_cases: Optional[str]
+    search_queries: Optional[Dict[str, List[str]]]
+    resources: Optional[List[dict]]
+    final_report: Optional[str]
+
+
+def clean_text(text):
+    """
+    Cleans the given text by:
+        1. Removing all hyperlinks.
+        2. Removing unnecessary parentheses and square brackets.
+
+    Args:
+        text (str): The input text to be cleaned.
+
+    Returns:
+        str: The cleaned text with hyperlinks, parentheses, and square brackets removed.
+    """
+    # Regular expression pattern for matching URLs
+    url_pattern = r'https?://\S+|www\.\S+'
+    # Remove hyperlinks
+    text_without_links = re.sub(url_pattern, '', text)
+    
+    # Regular expression pattern for matching parentheses and square brackets
+    brackets_pattern = r'[\[\]\(\)]'
+    # Remove unnecessary brackets
+    cleaned_text = re.sub(brackets_pattern, '', text_without_links)
+    
+    return cleaned_text.strip()
+
+
+def create_industry_research_workflow(llm):
+    async def industry_research(state: ResearchGraphState):
+        """Research industry and company using DeepWebCrawler."""
+        company = state['company']
+        industry = state['industry']
+        
+        queries = [
+            f"{company} company profile services",
+        ]
+        
+        crawler = DeepWebCrawler(
+            max_search_results=3,
+            max_external_links=1,
+            word_count_threshold=100,
+            content_filter_type='bm25',
+            filter_threshold=0.48
+        )
+        
+        all_results = []
+        for query in queries:
+            results = await crawler.search_and_crawl(query)
+            all_results.extend(results)
+        print(all_results)
+        combined_content = "\n\n".join([
+            f"Title: {clean_text(r['title'])} \n{clean_text(r['content'])}" 
+            for r in all_results if r['success']
+        ])
+        print("Combined Content: ", combined_content)
+        prompt = PromptTemplate.from_template(
+            """Analyze this research about {company} in the {industry} industry:
+            {content}
+            
+            Provide a comprehensive overview including:
+            1. Company Overview
+            2. Market Segments
+            3. Products and Services
+            4. Strategic Focus Areas
+            5. Industry Trends
+            6. Competitive Position
+            
+            Format the analysis in clear sections with headers."""
+        )
+        
+        chain = prompt | llm | StrOutputParser()
+        analysis = chain.invoke({
+            "company": company,
+            "industry": industry,
+            "content": combined_content
+        })
+        print("Analysis: ", analysis)
+        return {
+            "research_results": {
+                "analysis": analysis,
+                "raw_content": combined_content
+            }
+        }
+
+    def generate_use_cases_and_queries(state: ResearchGraphState):
+        """Generate AI/ML use cases and extract relevant search queries."""
+        research_data = state['research_results']
+        company = state['company']
+        industry = state['industry']
+        
+        # First generate use cases
+        use_case_prompt = PromptTemplate.from_template(
+            """Based on this research: 
+            
+            Analysis: {analysis}
+            Raw Research: {raw_content}
+            
+            Generate innovative use cases where {company} in the {industry} industry can leverage 
+            Generative AI and Large Language Models for:
+            
+            1. Internal Process Improvements
+            2. Customer Experience Enhancement
+            3. Product/Service Innovation
+            4. Data Analytics and Decision Making
+            
+            For each use case, provide:
+            - Clear description
+            - Expected benefits
+            - Implementation considerations"""
+        )
+        
+        chain = use_case_prompt | llm | StrOutputParser()
+        use_cases = chain.invoke({
+            "company": company,
+            "industry": industry,
+            "analysis": research_data['analysis'],
+            "raw_content": research_data['raw_content']
+        })
+        
+        # Then extract relevant search queries
+        query_extraction_prompt = PromptTemplate.from_template(
+            """Based on these AI/ML use cases for {company}:
+
+            {use_cases}
+            
+            Extract Two specific search queries for finding relevant datasets and implementations.
+            
+            Provide your response in this exact format:
+            DATASET QUERIES:
+            - query1
+            - query2
+
+            IMPLEMENTATION QUERIES:
+            - query1
+            - query2
+
+            Make queries specific and technical. Include ML model types, data types, and specific AI techniques."""
+        )
+        
+        chain = query_extraction_prompt | llm | StrOutputParser()
+        queries_text = chain.invoke({
+            "company": company,
+            "use_cases": use_cases
+        })
+        
+        # Parse the text response into structured format
+        def parse_queries(text):
+            dataset_queries = []
+            implementation_queries = []
+            current_section = None
+            
+            for line in text.split('\n'):
+                line = line.strip()
+                if line == "DATASET QUERIES:":
+                    current_section = "dataset"
+                elif line == "IMPLEMENTATION QUERIES:":
+                    current_section = "implementation"
+                elif line.startswith("- "):
+                    query = line[2:].strip()
+                    if current_section == "dataset":
+                        dataset_queries.append(query)
+                    elif current_section == "implementation":
+                        implementation_queries.append(query)
+            
+            return {
+                "dataset_queries": dataset_queries or ["machine learning datasets business", "AI training data industry"],
+                "implementation_queries": implementation_queries or ["AI tools business automation", "machine learning implementation"]
+            }
+
+        search_queries = parse_queries(queries_text)
+        print("Search_queries: ", search_queries)
+        return {
+            "use_cases": use_cases,
+            "search_queries": search_queries
+        }
+
+    async def collect_targeted_resources(state: ResearchGraphState):
+        """Find relevant datasets and resources using extracted queries."""
+        search_queries = state['search_queries']
+        resource_agent = ResourceCollectionAgent(max_results_per_query=5)
+        
+        # Collect resources using targeted queries
+        all_resources = {
+            "datasets": [],
+            "implementations": []
+        }
+        
+        # Search for datasets
+        for query in search_queries['dataset_queries']:
+            # Add platform-specific modifiers to queries
+            kaggle_query = f"site:kaggle.com/datasets {query}"
+            huggingface_query = f"site:huggingface.co/datasets {query}"
+            
+            resources = await resource_agent.collect_resources()
+            
+            # Process and categorize results
+            if resources.get("kaggle_datasets"):
+                all_resources["datasets"].extend([{
+                    "title": item["title"],
+                    "url": item["url"],
+                    "description": item["snippet"],
+                    "platform": "Kaggle",
+                    "query": query
+                } for item in resources["kaggle_datasets"]])
+                
+            if resources.get("huggingface_datasets"):
+                all_resources["datasets"].extend([{
+                    "title": item["title"],
+                    "url": item["url"],
+                    "description": item["snippet"],
+                    "platform": "HuggingFace",
+                    "query": query
+                } for item in resources["huggingface_datasets"]])
+        
+        # Search for implementations
+        for query in search_queries['implementation_queries']:
+            github_query = f"site:github.com {query}"
+            
+            resources = await resource_agent.collect_resources()
+            
+            if resources.get("github_repositories"):
+                all_resources["implementations"].extend([{
+                    "title": item["title"],
+                    "url": item["url"],
+                    "description": item["snippet"],
+                    "platform": "GitHub",
+                    "query": query
+                } for item in resources["github_repositories"]])
+        print("Resources: ", all_resources)
+        return {"resources": all_resources}
+
+    def generate_pdf_report(state: ResearchGraphState):
+        """Generate final PDF report with all collected information."""
+        research_data = state['research_results']
+        use_cases = state['use_cases']
+        resources = state['resources']
+        company = state['company']
+        industry = state['industry']
+        
+        # Format resources for manual append later
+        datasets_section = "\n## Available Datasets\n"
+        if resources.get('datasets'):
+            for dataset in resources['datasets']:
+                datasets_section += f"  - {dataset['platform']}: {dataset['url']}\n"
+        
+        implementations_section = "\n## Implementation Resources\n"
+        if resources.get('implementations'):
+            for impl in resources['implementations']:
+                implementations_section += f"  - {impl['platform']}: {impl['url']}\n"
+        
+        
+        prompt = PromptTemplate.from_template(
+        """
+        # GenAI & ML Implementation Proposal for {company}
+
+        ## Executive Summary
+        - **Current Position in the {industry} Industry**: 
+        - **Key Opportunities for AI/ML Implementation**: 
+        - **Expected Business Impact and ROI**: 
+        - **Implementation Timeline Overview**: 
+
+        ## Industry and Company Analysis
+        {analysis}
+
+        ## Strategic AI/ML Implementation Opportunities
+
+        Based on the analysis, here are the key opportunities for AI/ML implementation:
+
+        {use_cases}
+
+        Format the report in Markdown for clear sections, headings, and bullet points. Ensure professional formatting with structured subsections.
+        """
+        )
+        
+        chain = prompt | llm | StrOutputParser()
+        markdown_content = chain.invoke({
+            "company": company,
+            "industry": industry,
+            "analysis": research_data['analysis'],
+            "use_cases": use_cases,
+        })
+        
+        if markdown_content.startswith("```markdown") and markdown_content.endswith("```"):
+            markdown_content = markdown_content[len("```markdown"):].rstrip("```").strip()
+            
+        markdown_content += "\n\n" + datasets_section + "\n\n" + implementations_section
+        # Convert markdown to PDF
+        import tempfile
+        import os
+        import markdown2
+        from xhtml2pdf import pisa
+        
+        # Create temporary directory and full path for PDF
+        temp_dir = tempfile.mkdtemp()
+        pdf_filename = f"{company.replace(' ', '_')}_research_report.pdf"
+        pdf_path = os.path.join(temp_dir, pdf_filename)
+        
+        html_content = markdown2.markdown(markdown_content, extras=['tables', 'break-on-newline'])
+        # HTML template with enhanced styles (same as before)
+        html_template = f"""
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <meta charset="UTF-8">
+            <style>
+                @page {{
+                    size: A4;
+                    margin: 2.5cm;
+                    @frame footer {{
+                        -pdf-frame-content: footerContent;
+                        bottom: 1cm;
+                        margin-left: 1cm;
+                        margin-right: 1cm;
+                        height: 1cm;
+                    }}
+                }}
+                body {{
+                    font-family: Helvetica, Arial, sans-serif;
+                    font-size: 11pt;
+                    line-height: 1.6;
+                    color: #2c3e50;
+                }}
+                h1 {{
+                    font-size: 24pt;
+                    color: #1a237e;
+                    text-align: center;
+                    margin-bottom: 2cm;
+                    font-weight: bold;
+                }}
+                h2 {{
+                    font-size: 18pt;
+                    color: #283593;
+                    margin-top: 1.5cm;
+                    border-bottom: 2px solid #3949ab;
+                    padding-bottom: 0.3cm;
+                }}
+                h3 {{
+                    font-size: 14pt;
+                    color: #3949ab;
+                    margin-top: 1cm;
+                }}
+                h4 {{
+                    font-size: 12pt;
+                    color: #5c6bc0;
+                    margin-top: 0.8cm;
+                }}
+                p {{
+                    text-align: justify;
+                    margin-bottom: 0.5cm;
+                }}
+                ul {{
+                    margin-left: 0;
+                    padding-left: 1cm;
+                    margin-bottom: 0.5cm;
+                }}
+                li {{
+                    margin-bottom: 0.3cm;
+                }}
+                a {{
+                    color: #3f51b5;
+                    text-decoration: none;
+                }}
+                strong {{
+                    color: #283593;
+                }}
+                .use-case {{
+                    background-color: #f5f7fa;
+                    padding: 1cm;
+                    margin: 0.5cm 0;
+                    border-left: 4px solid #3949ab;
+                }}
+                .benefit {{
+                    margin-left: 1cm;
+                    color: #34495e;
+                }}
+            </style>
+        </head>
+        <body>
+            {html_content}
+            <div id="footerContent" style="text-align: center; font-size: 8pt; color: #7f8c8d;">
+                Page <pdf:pagenumber> of <pdf:pagecount>
+            </div>
+        </body>
+        </html>
+        """
+        
+        # Convert HTML to PDF with proper error handling
+        try:
+            with open(pdf_path, "w+b") as pdf_file:
+                result = pisa.CreatePDF(
+                    html_template,
+                    dest=pdf_file
+                )
+                if result.err:
+                    print(f"Error generating PDF: {result.err}")
+                    return {"final_report": None}
+                
+            # Verify the file exists
+            if os.path.exists(pdf_path):
+                print(f"PDF successfully generated at: {pdf_path}")
+                return {"final_report": pdf_path}
+            else:
+                print("PDF file was not created successfully")
+                return {"final_report": None}
+                
+        except Exception as e:
+            print(f"Exception during PDF generation: {str(e)}")
+            return {"final_report": None}
+
+    # Create workflow
+    workflow = StateGraph(ResearchGraphState)
+    
+    # Add nodes
+    workflow.add_node("industry_research", industry_research)
+    workflow.add_node("use_cases_gen", generate_use_cases_and_queries)
+    workflow.add_node("resources_gen", collect_targeted_resources)
+    workflow.add_node("report", generate_pdf_report)
+    
+    # Define edges
+    workflow.add_edge(START, "industry_research")
+    workflow.add_edge("industry_research", "use_cases_gen")
+    workflow.add_edge("use_cases_gen", "resources_gen")
+    workflow.add_edge("resources_gen", "report")
+    workflow.add_edge("report", END)
+    
+    return workflow.compile()
+
+async def run_industry_research(company: str, industry: str, llm):
+    """Run the industry research workflow asynchronously."""
+    workflow = create_industry_research_workflow(llm)
+    
+    final_state = None
+    output = await workflow.ainvoke(input={
+        "company": company,
+        "industry": industry
+    }, config={"recursion_limit": 5})
+    
+    return output['final_report']
+
+# Example usage
+if __name__ == "__main__":
+    async def main():
+        # Initialize LLM
+        llm = ChatOpenAI(
+        model="gpt-3.5-turbo-0125",
+        temperature=0.3,
+        timeout=None,
+        max_retries=2,)
+        
+        # Run the workflow
+        report_path = await run_industry_research(
+            company="Adani Defence & Aerospace",
+            industry="Defense Engineering and Construction",
+            llm=llm
+        )
+        print(f"Report generated at: {report_path}")
+    
+    asyncio.run(main())

src/agents/router.py

@@ -0,0 +1,59 @@
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_ollama import ChatOllama
+from pydantic import BaseModel, Field
+from typing import Literal
+
+class RouteQuery(BaseModel):
+    """Route a user query to the most relevant datasource."""
+    datasource: Literal["vectorstore", "web_search"] = Field(
+        description="Route question to web search or vectorstore retrieval"
+    )
+
+def create_query_router():
+    """
+    Create a query router to determine data source for a given question.
+    
+    Returns:
+        Callable: Query router function
+    """
+    # LLM with function call
+    llm = ChatOllama(model = "llama3.2", temperature = 0.1, num_predict = 256, top_p=0.5)
+    structured_llm_router = llm.with_structured_output(RouteQuery)
+
+    # Prompt
+    system = """You are an expert at routing a user question to a vectorstore or web search.
+    The vectorstore contains documents related to agents, prompt engineering, and adversarial attacks.
+    Use the vectorstore for questions on these topics. Otherwise, use web-search."""
+    
+    route_prompt = ChatPromptTemplate.from_messages([
+        ("system", system),
+        ("human", "{question}"),
+    ])
+
+    return route_prompt | structured_llm_router
+
+def route_query(question: str):
+    """
+    Route a specific query to its appropriate data source.
+    
+    Args:
+        question (str): User's input question
+    
+    Returns:
+        str: Recommended data source
+    """
+    router = create_query_router()
+    result = router.invoke({"question": question})
+    return result.datasource
+
+if __name__ == "__main__":
+    # Example usage
+    test_questions = [
+        "Who will the Bears draft first in the NFL draft?",
+        "What are the types of agent memory?"
+    ]
+    
+    for q in test_questions:
+        source = route_query(q)
+        print(f"Question: {q}")
+        print(f"Routed to: {source}\n")

src/agents/state.py

@@ -0,0 +1,30 @@
+from typing import List, TypedDict
+from langchain_core.documents.base import Document
+
+class GraphState(TypedDict):
+    """
+    Represents the state of our adaptive RAG graph.
+
+    Attributes:
+        question (str): Original user question
+        generation (str, optional): LLM generated answer
+        documents (List[Document], optional): Retrieved or searched documents
+    """
+    question: str
+    generation: str | None
+    documents: List[Document]
+
+class ResearchGraphState(TypedDict):
+    """
+    Represents the state of our adaptive RAG graph.
+
+    Attributes:
+        question (str): Original user question
+        generation (str, optional): LLM generated answer
+        documents (List[Document], optional): Retrieved or searched documents
+    """
+    company: str
+    industry: str | None
+    research_results: str | None
+    use_cases: str | None
+    search_queries: str | None

src/agents/workflow.py

@@ -0,0 +1,274 @@
+from langgraph.graph import END, StateGraph, START
+from langchain_core.prompts import PromptTemplate
+from src.agents.state import GraphState
+# from agents.router import route_query
+import asyncio
+from src.vectorstore.pinecone_db import get_retriever
+from src.tools.web_search import AdvancedWebCrawler
+from src.llm.graders import (
+    grade_document_relevance, 
+    check_hallucination, 
+    grade_answer_quality
+)
+from langchain_core.output_parsers import StrOutputParser
+from src.llm.query_rewriter import rewrite_query
+from langchain_ollama import ChatOllama
+
+def perform_web_search(question: str):
+    """
+    Perform web search using the AdvancedWebCrawler.
+    
+    Args:
+        question (str): User's input question
+    
+    Returns:
+        List: Web search results
+    """
+    # Initialize web crawler
+    crawler = AdvancedWebCrawler(
+        max_search_results=5,
+        word_count_threshold=50,
+        content_filter_type='f',
+        filter_threshold=0.48
+    )
+    results = asyncio.run(crawler.search_and_crawl(question))
+    
+    return results
+
+
+def create_adaptive_rag_workflow(retriever, llm, top_k=5, enable_websearch=False):
+    """
+    Create the adaptive RAG workflow graph.
+    
+    Args:
+        retriever: Vector store retriever
+    
+    Returns:
+        Compiled LangGraph workflow
+    """
+    def retrieve(state: GraphState):
+        """Retrieve documents from vectorstore."""
+        print("---RETRIEVE---")
+        question = state['question']
+        documents = retriever.invoke(question, top_k)
+        print(f"Retrieved {len(documents)} documents.")
+        print(documents)
+        return {"documents": documents, "question": question}
+
+    def route_to_datasource(state: GraphState):
+        """Route question to web search or vectorstore."""
+        print("---ROUTE QUESTION---")
+        # question = state['question']
+        # source = route_query(question)
+       
+        if enable_websearch:
+            print("---ROUTE TO WEB SEARCH---")
+            return "web_search"
+        else:
+            print("---ROUTE TO RAG---")
+            return "vectorstore"
+
+    def generate_answer(state: GraphState):
+        """Generate answer using retrieved documents."""
+        print("---GENERATE---")
+        question = state['question']
+        documents = state['documents']
+        
+        # Prepare context
+        context = "\n\n".join([doc["page_content"] for doc in documents])
+        prompt_template = PromptTemplate.from_template("""You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.
+        Question: {question}
+        Context: {context}
+        Answer:""")
+        # Generate answer
+        rag_chain = prompt_template | llm | StrOutputParser()
+
+        generation = rag_chain.invoke({"context": context, "question": question})
+        
+        return {"generation": generation, "documents": documents, "question": question}
+
+    def grade_documents(state: GraphState):
+        """Filter relevant documents."""
+        print("---GRADE DOCUMENTS---")
+        question = state['question']
+        documents = state['documents']
+        
+        # Filter documents
+        filtered_docs = []
+        for doc in documents:
+            score = grade_document_relevance(question, doc["page_content"], llm)
+            if score == "yes":
+                filtered_docs.append(doc)
+        
+        return {"documents": filtered_docs, "question": question}
+
+    def web_search(state: GraphState):
+        """Perform web search."""
+        print("---WEB SEARCH---")
+        question = state['question']
+        
+        # Perform web search
+        results = perform_web_search(question)
+        web_documents = [
+            {
+                "page_content": result['content'], 
+                "metadata": {"source": result['url']}
+            } for result in results
+        ]
+        
+        return {"documents": web_documents, "question": question}
+
+    def check_generation_quality(state: GraphState):
+        """Check the quality of generated answer."""
+        print("---ASSESS GENERATION---")
+        question = state['question']
+        documents = state['documents']
+        generation = state['generation']
+ 
+        
+        print("---Generation is not hallucinated.---")
+        # Check answer quality
+        quality_score = grade_answer_quality(question, generation, llm)
+        if quality_score == "yes":
+            print("---Answer quality is good.---")
+        else:
+            print("---Answer quality is poor.---")
+        return "end" if quality_score == "yes" else "rewrite"
+
+    # Create workflow
+    workflow = StateGraph(GraphState)
+
+    # Add nodes
+    workflow.add_node("vectorstore", retrieve)
+    workflow.add_node("web_search", web_search)
+    workflow.add_node("grade_documents", grade_documents)
+    workflow.add_node("generate", generate_answer)
+    workflow.add_node("rewrite_query", lambda state: {
+        "question": rewrite_query(state['question'], llm),
+        "documents": [],
+        "generation": None
+    })
+
+    # Define edges
+    workflow.add_conditional_edges(
+        START, 
+        route_to_datasource,
+        {
+            "web_search": "web_search",
+            "vectorstore": "vectorstore"
+        }
+    )
+    
+    workflow.add_edge("web_search", "generate")
+    workflow.add_edge("vectorstore", "grade_documents")
+    
+    workflow.add_conditional_edges(
+        "grade_documents",
+        lambda state: "generate" if state['documents'] else "rewrite_query"
+    )
+    
+    workflow.add_edge("rewrite_query", "vectorstore")
+    
+    workflow.add_conditional_edges(
+        "generate",
+        check_generation_quality,
+        {
+            "end": END,
+            "regenerate": "generate",
+            "rewrite": "rewrite_query"
+        }
+    )
+
+    # Compile the workflow
+    app = workflow.compile()
+    return app
+
+def run_adaptive_rag(retriever, question: str, llm, top_k=5, enable_websearch=False):
+    """
+    Run the adaptive RAG workflow for a given question.
+    
+    Args:
+        retriever: Vector store retriever
+        question (str): User's input question
+    
+    Returns:
+        str: Generated answer
+    """
+    # Create workflow
+    workflow = create_adaptive_rag_workflow(retriever, llm, top_k, enable_websearch=enable_websearch)
+    
+    # Run workflow
+    final_state = None
+    for output in workflow.stream({"question": question}, config={"recursion_limit": 5}):
+        for key, value in output.items():
+            print(f"Node '{key}':")
+            # Optionally print state details
+            # print(value)
+        final_state = value
+    
+    return final_state.get('generation', 'No answer could be generated.')
+
+if __name__ == "__main__":
+    # Example usage
+    from vectorstore.pinecone_db import PINECONE_API_KEY, ingest_data,  get_retriever, load_documents, process_chunks, save_to_parquet
+    from pinecone import Pinecone
+    
+    # Load and prepare documents
+    pc = Pinecone(api_key=PINECONE_API_KEY)
+    
+    # Define input files
+    file_paths=[
+        # './data/2404.19756v1.pdf',
+        # './data/OD429347375590223100.pdf',
+        # './data/Project Report Format.docx',
+        './data/UNIT 2 GENDER BASED VIOLENCE.pptx'
+    ]
+
+    # Process pipeline
+    try:
+        # Step 1: Load and combine documents
+        print("Loading documents...")
+        markdown_path = load_documents(file_paths)
+        
+        # Step 2: Process into chunks with embeddings
+        print("Processing chunks...")
+        chunks = process_chunks(markdown_path)
+        
+        # Step 3: Save to Parquet
+        print("Saving to Parquet...")
+        parquet_path = save_to_parquet(chunks)
+        
+        # Step 4: Ingest into Pinecone
+        print("Ingesting into Pinecone...")
+        ingest_data(pc,
+            parquet_path=parquet_path,
+            text_column="text",
+            pinecone_client=pc,
+        )
+        
+        # Step 5: Test retrieval
+        print("\nTesting retrieval...")
+        retriever = get_retriever(
+            pinecone_client=pc,
+            index_name="vector-index",
+            namespace="rag"
+        )
+        
+    except Exception as e:
+        print(f"Error in pipeline: {str(e)}")    
+
+    llm = ChatOllama(model = "llama3.2", temperature = 0.1, num_predict = 256, top_p=0.5)
+    
+    # Test questions
+    test_questions = [
+        # "What are the key components of AI agent memory?",
+        # "Explain prompt engineering techniques",
+        # "What are recent advancements in adversarial attacks on LLMs?"
+        "what are the trending papers that are published in NeurIPS 2024?"
+    ]
+    
+    # Run workflow for each test question
+    for question in test_questions:
+        print(f"\n--- Processing Question: {question} ---")
+        answer = run_adaptive_rag(retriever, question, llm)
+        print("\nFinal Answer:", answer)

src/data_processing/chunker.py

@@ -0,0 +1,85 @@
+from typing import List
+import numpy as np
+from chonkie.embeddings import BaseEmbeddings
+from FlagEmbedding import BGEM3FlagModel
+from chonkie import SDPMChunker as SDPMChunker
+
+class BGEM3Embeddings(BaseEmbeddings):
+    def __init__(self, model_name):
+        self.model = BGEM3FlagModel(model_name, use_fp16=True)
+        self.task = "separation"
+    
+    @property
+    def dimension(self):
+        return 1024
+
+    def embed(self, text: str):
+        e = self.model.encode([text], return_dense=True, return_sparse=False, return_colbert_vecs=False)['dense_vecs']
+        # print(e)
+        return e
+
+    def embed_batch(self, texts: List[str]):
+        embeddings = self.model.encode(texts, return_dense=True, return_sparse=False, return_colbert_vecs=False
+        )
+        # print(embeddings['dense_vecs'])
+        return embeddings['dense_vecs']
+
+    def count_tokens(self, text: str):
+        l = len(self.model.tokenizer.encode(text))
+        # print(l)
+        return l
+
+    def count_tokens_batch(self, texts: List[str]):
+        encodings = self.model.tokenizer(texts)
+        # print([len(enc) for enc in encodings["input_ids"]])
+        return [len(enc) for enc in encodings["input_ids"]]
+
+    def get_tokenizer_or_token_counter(self):
+        return self.model.tokenizer
+    
+    def similarity(self, u: "np.ndarray", v: "np.ndarray"):
+        """Compute cosine similarity between two embeddings."""
+        s = ([email protected])#.item()
+        # print(s)
+        return s
+    
+    @classmethod
+    def is_available(cls):
+        return True
+
+    def __repr__(self):
+        return "bgem3"
+
+
+def main():
+    # Initialize the BGE M3 embeddings model
+    embedding_model = BGEM3Embeddings(
+        model_name="BAAI/bge-m3"
+    )
+
+    # Initialize the SDPM chunker
+    chunker = SDPMChunker(
+        embedding_model=embedding_model,
+        chunk_size=256,
+        threshold=0.7,
+        skip_window=2
+    )
+
+    with open('./output.md', 'r') as file:
+        text = file.read()
+
+    # Generate chunks
+    chunks = chunker.chunk(text)
+
+    # Print the chunks
+    for i, chunk in enumerate(chunks, 1):
+        print(f"\nChunk {i}:")
+        print(f"Text: {chunk.text}")
+        print(f"Token count: {chunk.token_count}")
+        print(f"Start index: {chunk.start_index}")
+        print(f"End index: {chunk.end_index}")
+        print(f"no of sentences: {len(chunk.sentences)}")
+        print("-" * 80)
+
+if __name__ == "__main__":
+    main()

src/data_processing/loader.py

@@ -0,0 +1,153 @@
+from pathlib import Path
+from typing import List, Union
+import logging
+from dataclasses import dataclass
+
+from langchain_core.documents import Document as LCDocument
+from langchain_core.document_loaders import BaseLoader
+from docling.document_converter import DocumentConverter, PdfFormatOption
+from docling.datamodel.base_models import InputFormat, ConversionStatus
+from docling.datamodel.pipeline_options import (
+    PdfPipelineOptions,
+    EasyOcrOptions
+)
+
+logging.basicConfig(level=logging.INFO)
+_log = logging.getLogger(__name__)
+
+@dataclass
+class ProcessingResult:
+    """Store results of document processing"""
+    success_count: int = 0
+    failure_count: int = 0
+    partial_success_count: int = 0
+    failed_files: List[str] = None
+
+    def __post_init__(self):
+        if self.failed_files is None:
+            self.failed_files = []
+
+class MultiFormatDocumentLoader(BaseLoader):
+    """Loader for multiple document formats that converts to LangChain documents"""
+    
+    def __init__(
+        self,
+        file_paths: Union[str, List[str]],
+        enable_ocr: bool = True,
+        enable_tables: bool = True
+    ):
+        self._file_paths = [file_paths] if isinstance(file_paths, str) else file_paths
+        self._enable_ocr = enable_ocr
+        self._enable_tables = enable_tables
+        self._converter = self._setup_converter()
+        
+    def _setup_converter(self):
+        """Set up the document converter with appropriate options"""
+        # Configure pipeline options
+        pipeline_options = PdfPipelineOptions(do_ocr=False, do_table_structure=False, ocr_options=EasyOcrOptions(
+                force_full_page_ocr=True
+            ))
+        if self._enable_ocr:
+            pipeline_options.do_ocr = True
+        if self._enable_tables:
+            pipeline_options.do_table_structure = True
+            pipeline_options.table_structure_options.do_cell_matching = True
+
+        # Create converter with supported formats
+        return DocumentConverter(
+            allowed_formats=[
+                InputFormat.PDF,
+                InputFormat.IMAGE,
+                InputFormat.DOCX,
+                InputFormat.HTML,
+                InputFormat.PPTX,
+                InputFormat.ASCIIDOC,
+                InputFormat.MD,
+            ],
+            format_options={
+            InputFormat.PDF: PdfFormatOption(
+                pipeline_options=pipeline_options,
+            )}
+        )
+
+    def lazy_load(self):
+        """Convert documents and yield LangChain documents"""
+        results = ProcessingResult()
+        
+        for file_path in self._file_paths:
+            try:
+                path = Path(file_path)
+                if not path.exists():
+                    _log.warning(f"File not found: {file_path}")
+                    results.failure_count += 1
+                    results.failed_files.append(file_path)
+                    continue
+
+                conversion_result = self._converter.convert(path)
+                
+                if conversion_result.status == ConversionStatus.SUCCESS:
+                    results.success_count += 1
+                    text = conversion_result.document.export_to_markdown()
+                    metadata = {
+                        'source': str(path),
+                        'file_type': path.suffix,
+                    }
+                    yield LCDocument(
+                        page_content=text,
+                        metadata=metadata
+                    )
+                elif conversion_result.status == ConversionStatus.PARTIAL_SUCCESS:
+                    results.partial_success_count += 1
+                    _log.warning(f"Partial conversion for {file_path}")
+                    text = conversion_result.document.export_to_markdown()
+                    metadata = {
+                        'source': str(path),
+                        'file_type': path.suffix,
+                        'conversion_status': 'partial'
+                    }
+                    yield LCDocument(
+                        page_content=text,
+                        metadata=metadata
+                    )
+                else:
+                    results.failure_count += 1
+                    results.failed_files.append(file_path)
+                    _log.error(f"Failed to convert {file_path}")
+                    
+            except Exception as e:
+                _log.error(f"Error processing {file_path}: {str(e)}")
+                results.failure_count += 1
+                results.failed_files.append(file_path)
+
+        # Log final results
+        total = results.success_count + results.partial_success_count + results.failure_count
+        _log.info(
+            f"Processed {total} documents:\n"
+            f"- Successfully converted: {results.success_count}\n"
+            f"- Partially converted: {results.partial_success_count}\n"
+            f"- Failed: {results.failure_count}"
+        )
+        if results.failed_files:
+            _log.info("Failed files:")
+            for file in results.failed_files:
+                _log.info(f"- {file}")
+                
+                
+if __name__ == '__main__':
+    # Load documents from a list of file paths
+    loader = MultiFormatDocumentLoader(
+        file_paths=[
+            # './data/2404.19756v1.pdf',
+            # './data/OD429347375590223100.pdf',
+            './data/Project Report Format.docx',
+            # './data/UNIT 2 GENDER BASED VIOLENCE.pptx'
+        ],
+        enable_ocr=False,
+        enable_tables=True
+    )
+    for doc in loader.lazy_load():
+        print(doc.page_content)
+        print(doc.metadata)
+        # save document in .md file 
+        with open('output.md', 'w') as f:
+            f.write(doc.page_content)

src/llm/graders.py

@@ -0,0 +1,128 @@
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_ollama import ChatOllama
+from pydantic import BaseModel, Field
+from typing import List
+
+class DocumentRelevance(BaseModel):
+    """Binary score for relevance check on retrieved documents."""
+    binary_score: str = Field(
+        description="Documents are relevant to the question, 'yes' or 'no'"
+    )
+
+class HallucinationCheck(BaseModel):
+    """Binary score for hallucination present in generation answer."""
+    binary_score: str = Field(
+        description="Answer is grounded in the facts, 'yes' or 'no'"
+    )
+
+class AnswerQuality(BaseModel):
+    """Binary score to assess answer addresses question."""
+    binary_score: str = Field(
+        description="Answer addresses the question, 'yes' or 'no'"
+    )
+
+def create_llm_grader(grader_type: str, llm):
+    """
+    Create an LLM grader based on the specified type.
+    
+    Args:
+        grader_type (str): Type of grader to create
+    
+    Returns:
+        Callable: LLM grader function
+    """
+    # Initialize LLM
+    
+    # Select grader type and create structured output
+    if grader_type == "document_relevance":
+        structured_llm_grader = llm.with_structured_output(DocumentRelevance)
+        system = """You are a grader assessing relevance of a retrieved document to a user question. 
+        If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. 
+        It does not need to be a stringent test. The goal is to filter out erroneous retrievals. 
+        Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question."""
+        
+        prompt = ChatPromptTemplate.from_messages([
+            ("system", system),
+            ("human", "Retrieved document: \n\n {document} \n\n User question: {question}"),
+        ])
+        
+    elif grader_type == "hallucination":
+        structured_llm_grader = llm.with_structured_output(HallucinationCheck)
+        system = """You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. 
+        Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts."""
+        
+        prompt = ChatPromptTemplate.from_messages([
+            ("system", system),
+            ("human", "Set of facts: \n\n {documents} \n\n LLM generation: {generation}"),
+        ])
+        
+    elif grader_type == "answer_quality":
+        structured_llm_grader = llm.with_structured_output(AnswerQuality)
+        system = """You are a grader assessing whether an answer addresses / resolves a question. 
+        Give a binary score 'yes' or 'no'. 'Yes' means that the answer resolves the question."""
+        
+        prompt = ChatPromptTemplate.from_messages([
+            ("system", system),
+            ("human", "User question: \n\n {question} \n\n LLM generation: {generation}"),
+        ])
+    
+    else:
+        raise ValueError(f"Unknown grader type: {grader_type}")
+    
+    return prompt | structured_llm_grader
+
+def grade_document_relevance(question: str, document: str, llm):
+    """
+    Grade the relevance of a document to a given question.
+    
+    Args:
+        question (str): User's question
+        document (str): Retrieved document content
+    
+    Returns:
+        str: Binary score ('yes' or 'no')
+    """
+    grader = create_llm_grader("document_relevance", llm)
+    result = grader.invoke({"question": question, "document": document})
+    return result.binary_score
+
+def check_hallucination(documents: List[str], generation: str, llm):
+    """
+    Check if the generation is grounded in the provided documents.
+    
+    Args:
+        documents (List[str]): List of source documents
+        generation (str): LLM generated answer
+    
+    Returns:
+        str: Binary score ('yes' or 'no')
+    """
+    grader = create_llm_grader("hallucination", llm)
+    result = grader.invoke({"documents": documents, "generation": generation})
+    return result.binary_score
+
+def grade_answer_quality(question: str, generation: str, llm):
+    """
+    Grade the quality of the answer in addressing the question.
+    
+    Args:
+        question (str): User's original question
+        generation (str): LLM generated answer
+    
+    Returns:
+        str: Binary score ('yes' or 'no')
+    """
+    grader = create_llm_grader("answer_quality", llm)
+    result = grader.invoke({"question": question, "generation": generation})
+    return result.binary_score
+
+if __name__ == "__main__":
+    # Example usage
+    test_question = "What are the types of agent memory?"
+    test_document = "Agent memory can be classified into different types such as episodic, semantic, and working memory."
+    test_generation = "Agent memory includes episodic memory for storing experiences, semantic memory for general knowledge, and working memory for immediate processing."
+    llm = ChatOllama(model = "llama3.2", temperature = 0.1, num_predict = 256, top_p=0.5)
+    
+    print("Document Relevance:", grade_document_relevance(test_question, test_document, llm))
+    print("Hallucination Check:", check_hallucination([test_document], test_generation, llm))
+    print("Answer Quality:", grade_answer_quality(test_question, test_generation, llm))

src/llm/query_rewriter.py

@@ -0,0 +1,55 @@
+from langchain_core.prompts.chat import ChatPromptTemplate
+from langchain_ollama import ChatOllama
+from langchain_core.output_parsers import StrOutputParser
+
+def create_query_rewriter(llm):
+    """
+    Create a query rewriter to optimize retrieval.
+    
+    Returns:
+        Callable: Query rewriter function
+    """
+    
+    # Prompt for query rewriting
+    system = """You are a question re-writer that converts an input question to a better version that is optimized 
+    for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning."""
+    
+    re_write_prompt = ChatPromptTemplate.from_messages([
+        ("system", system),
+        ("human", "Here is the initial question: \n\n {question} \n Formulate an improved question."),
+    ])
+    
+    # Create query rewriter chain
+    return re_write_prompt | llm | StrOutputParser()
+
+def rewrite_query(question: str, llm):
+    """
+    Rewrite a given query to optimize retrieval.
+    
+    Args:
+        question (str): Original user question
+    
+    Returns:
+        str: Rewritten query
+    """
+    query_rewriter = create_query_rewriter(llm)
+    try:
+        rewritten_query = query_rewriter.invoke({"question": question})
+        return rewritten_query
+    except Exception as e:
+        print(f"Query rewriting error: {e}")
+        return question
+
+if __name__ == "__main__":
+    # Example usage
+    test_queries = [
+        "Tell me about AI agents",
+        "What do we know about memory in AI systems?",
+        "Bears draft strategy"
+    ]
+    llm = ChatOllama(model = "llama3.2", temperature = 0.1, num_predict = 256, top_p=0.5)
+    
+    for query in test_queries:
+        rewritten = rewrite_query(query, llm)
+        print(f"Original: {query}")
+        print(f"Rewritten: {rewritten}\n")

src/tools/deep_crawler.py

@@ -0,0 +1,325 @@
+import os
+import sys
+import asyncio
+from typing import List, Dict, Optional, Set
+from urllib.parse import urlparse
+from langchain_community.tools import DuckDuckGoSearchResults, TavilySearchResults
+from langchain_community.utilities.duckduckgo_search import DuckDuckGoSearchAPIWrapper
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class DeepWebCrawler:
+    def __init__(self, 
+                 max_search_results: int = 5,
+                 max_external_links: int = 3,
+                 word_count_threshold: int = 50,
+                 content_filter_type: str = 'pruning',
+                 filter_threshold: float = 0.48):
+        """
+        Initialize the Deep Web Crawler with support for one-level deep crawling
+        
+        Args:
+            max_search_results (int): Maximum number of search results to process
+            max_external_links (int): Maximum number of external links to crawl per page
+            word_count_threshold (int): Minimum word count for crawled content
+            content_filter_type (str): Type of content filter ('pruning' or 'bm25')
+            filter_threshold (float): Threshold for content filtering
+        """
+        self.max_search_results = max_search_results
+        self.max_external_links = max_external_links
+        self.word_count_threshold = word_count_threshold
+        self.content_filter_type = content_filter_type
+        self.filter_threshold = filter_threshold
+        self.crawled_urls: Set[str] = set()
+
+    def _create_web_search_tool(self):
+        return TavilySearchResults(max_results=self.max_search_results)
+
+    def _create_content_filter(self, user_query: Optional[str] = None):
+        if self.content_filter_type == 'bm25' and user_query:
+            return BM25ContentFilter(
+                user_query=user_query, 
+                bm25_threshold=self.filter_threshold
+            )
+        return PruningContentFilter(
+            threshold=self.filter_threshold,
+            threshold_type="fixed",
+            min_word_threshold=self.word_count_threshold
+        )
+
+    def _extract_links_from_search_results(self, results: List[Dict]) -> List[str]:
+        """Safely extract URLs from search results"""
+        urls = []
+        for result in results:
+            if isinstance(result, dict) and 'url' in result:
+                urls.append(result['url'])
+            elif isinstance(result, str):
+                urls.append(result)
+        return urls
+
+    def _extract_url_from_link(self, link):
+        """Extract URL string from link object which might be a dict or string"""
+        if isinstance(link, dict):
+            return link.get('url', '')  # Assuming the URL is stored in a 'url' key
+        elif isinstance(link, str):
+            return link
+        return ''
+    
+    def _process_crawl_result(self, result) -> Dict:
+        """Process individual crawl result into structured format"""
+        return {
+            "url": result.url,
+            "success": result.success,
+            "title": result.metadata.get('title', 'N/A'),
+            "content": result.markdown_v2.raw_markdown if result.success else result.error_message,
+            "word_count": len(result.markdown_v2.raw_markdown.split()) if result.success else 0,
+            "links": {
+                "internal": result.links.get('internal', []),
+                "external": result.links.get('external', [])
+            },
+            "images": len(result.media.get('images', []))
+        }
+
+    async def crawl_urls(self, urls: List[str], user_query: Optional[str] = None, depth: int = 0):
+        """
+        Crawl URLs with support for external link crawling
+        
+        Args:
+            urls (List[str]): List of URLs to crawl
+            user_query (Optional[str]): Query for content filtering
+            depth (int): Current crawl depth (0 for initial, 1 for external links)
+        
+        Returns:
+            List of crawl results including external link content
+        """
+        if not urls or depth > 1:
+            return []
+
+        # Filter out already crawled URLs
+        new_urls = [url for url in urls if url not in self.crawled_urls]
+        if not new_urls:
+            return []
+
+        async with AsyncWebCrawler(
+            browser_type="chromium",
+            headless=True,
+            verbose=True
+        ) as crawler:
+            content_filter = self._create_content_filter(user_query)
+            
+            results = await crawler.arun_many(
+                urls=new_urls,
+                word_count_threshold=self.word_count_threshold,
+                cache_mode=CacheMode.BYPASS,
+                markdown_generator=DefaultMarkdownGenerator(content_filter=content_filter),
+                exclude_external_links=True,
+                exclude_social_media_links=True,
+                remove_overlay_elements=True,
+                simulate_user=True,
+                magic=True
+            )
+
+            processed_results = []
+            external_urls = set()
+
+            # Process results and collect external URLs
+            for result in results:
+                self.crawled_urls.add(result.url)
+                processed_result = self._process_crawl_result(result)
+                processed_results.append(processed_result)
+
+                if depth == 0 and result.success:
+                    # Collect unique external URLs for further crawling
+                    external_links = result.links.get('external', [])[:self.max_external_links]
+                    external_urls.update(
+                        self._extract_url_from_link(link) 
+                        for link in external_links 
+                        if self._extract_url_from_link(link) 
+                        and self._extract_url_from_link(link) not in self.crawled_urls
+                    )
+
+            # Crawl external links if at depth 0
+            if depth == 0 and external_urls and False:
+                external_results = await self.crawl_urls(
+                    list(external_urls),
+                    user_query=user_query,
+                    depth=0
+                )
+                processed_results.extend(external_results)
+
+            return processed_results
+
+    async def search_and_crawl(self, query: str) -> List[Dict]:
+        """
+        Perform web search and deep crawl of results
+        
+        Args:
+            query (str): Search query
+        
+        Returns:
+            List of crawled content results including external links
+        """
+
+        search_tool = self._create_web_search_tool()
+        search_results = search_tool.invoke(query)
+        
+        # Handle different types of search results
+        if isinstance(search_results, str):
+            urls = [search_results]
+        elif isinstance(search_results, list):
+            urls = self._extract_links_from_search_results(search_results)
+        else:
+            print(f"Unexpected search results format: {type(search_results)}")
+            return []
+        
+        if not urls:
+            print("No valid URLs found in search results")
+            return []
+        
+        print(f"Initial search found {len(urls)} URLs for query: {query}")
+        print(urls)
+        crawl_results = await self.crawl_urls(urls, user_query=query)
+        
+        return crawl_results
+
+
+class ResourceCollectionAgent:
+    def __init__(self, max_results_per_query: int = 10):
+        """
+        Initialize the Resource Collection Agent
+        
+        Args:
+            max_results_per_query (int): Maximum number of results per search query
+        """
+        self.max_results_per_query = max_results_per_query
+        self.search_tool = TavilySearchResults(max_results=max_results_per_query)
+
+    def _is_valid_domain(self, url: str, valid_domains: List[str]) -> bool:
+        """Check if URL belongs to allowed domains"""
+        try:
+            domain = urlparse(url).netloc.lower()
+            return any(valid_domain in domain for valid_domain in valid_domains)
+        except:
+            return False
+
+    def _extract_search_result(self, result) -> Optional[Dict]:
+        """Safely extract information from a search result"""
+        try:
+            if isinstance(result, dict):
+                return {
+                    "title": result.get("title", "No title"),
+                    "url": result.get("url", ""),
+                    "snippet": result.get("snippet", "No description")
+                }
+            elif isinstance(result, str):
+                return {
+                    "title": "Unknown",
+                    "url": result,
+                    "snippet": "No description available"
+                }
+            return None
+        except Exception as e:
+            print(f"Error processing search result: {str(e)}")
+            return None
+
+    async def collect_resources(self) -> Dict[str, List[Dict]]:
+        """
+        Collect AI/ML resources from specific platforms
+        
+        Returns:
+            Dictionary with categorized resource links
+        """
+        search_queries = {
+            "datasets": [
+                ("kaggle", "site:kaggle.com/datasets machine learning"),
+                ("huggingface", "site:huggingface.co/datasets artificial intelligence")
+            ],
+            "repositories": [
+                ("github", "site:github.com AI tools repository")
+            ]
+        }
+
+        results = {
+            "kaggle_datasets": [],
+            "huggingface_datasets": [],
+            "github_repositories": []
+        }
+
+        for category, queries in search_queries.items():
+            for platform, query in queries:
+                try:
+                    search_results = self.search_tool.invoke(query)
+                    
+                    # Handle different result formats
+                    if isinstance(search_results, str):
+                        search_results = [search_results]
+                    elif not isinstance(search_results, list):
+                        print(f"Unexpected search results format for {platform}: {type(search_results)}")
+                        continue
+                    
+                    # Filter results based on domain
+                    valid_domains = {
+                        "kaggle": ["kaggle.com"],
+                        "huggingface": ["huggingface.co"],
+                        "github": ["github.com"]
+                    }
+                    
+                    for result in search_results:
+                        processed_result = self._extract_search_result(result)
+                        if processed_result and self._is_valid_domain(
+                            processed_result["url"], 
+                            valid_domains[platform]
+                        ):
+                            if platform == "kaggle":
+                                results["kaggle_datasets"].append(processed_result)
+                            elif platform == "huggingface":
+                                results["huggingface_datasets"].append(processed_result)
+                            elif platform == "github":
+                                results["github_repositories"].append(processed_result)
+                    
+                except Exception as e:
+                    print(f"Error collecting {platform} resources: {str(e)}")
+                    continue
+
+        return results
+
+def main():
+    async def run_examples():
+        # Test DeepWebCrawler
+        deep_crawler = DeepWebCrawler(
+            max_search_results=3,
+            max_external_links=2,
+            word_count_threshold=50
+        )
+        
+        crawl_results = await deep_crawler.search_and_crawl(
+            "Adani Defence & Aerospace"
+        )
+        
+        print("\nDeep Crawler Results:")
+        for result in crawl_results:
+            print(f"URL: {result['url']}")
+            print(f"Title: {result['title']}")
+            print(f"Word Count: {result['word_count']}")
+            print(f"External Links: {len(result['links']['external'])}\n")
+
+        # Test ResourceCollectionAgent
+        resource_agent = ResourceCollectionAgent(max_results_per_query=5)
+        resources = await resource_agent.collect_resources()
+        
+        print("\nResource Collection Results:")
+        for category, items in resources.items():
+            print(f"\n{category.upper()}:")
+            for item in items:
+                print(f"Title: {item['title']}")
+                print(f"URL: {item['url']}")
+                print("---")
+
+    asyncio.run(run_examples())
+
+if __name__ == "__main__":
+    main()

src/tools/web_search.py

@@ -0,0 +1,173 @@
+import os
+import sys
+import asyncio
+from typing import List, Dict, Optional
+
+from langchain_community.tools import DuckDuckGoSearchResults
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class AdvancedWebCrawler:
+    def __init__(self, 
+                 max_search_results: int = 5, 
+                 word_count_threshold: int = 50,
+                 content_filter_type: str = 'pruning',
+                 filter_threshold: float = 0.48):
+        """
+        Initialize the Advanced Web Crawler
+        
+        Args:
+            max_search_results (int): Maximum number of search results to process
+            word_count_threshold (int): Minimum word count for crawled content
+            content_filter_type (str): Type of content filter ('pruning' or 'bm25')
+            filter_threshold (float): Threshold for content filtering
+        """
+        self.max_search_results = max_search_results
+        self.word_count_threshold = word_count_threshold
+        self.content_filter_type = content_filter_type
+        self.filter_threshold = filter_threshold
+
+    def _create_web_search_tool(self):
+        """
+        Create a web search tool using DuckDuckGo
+        
+        Returns:
+            DuckDuckGoSearchResults: Web search tool
+        """
+        return DuckDuckGoSearchResults(max_results=self.max_search_results, output_format="list")
+
+    def _create_content_filter(self, user_query: Optional[str] = None):
+        """
+        Create content filter based on specified type
+        
+        Args:
+            user_query (Optional[str]): Query to use for BM25 filtering
+        
+        Returns:
+            Content filter strategy
+        """
+        if self.content_filter_type == 'bm25' and user_query:
+            return BM25ContentFilter(
+                user_query=user_query, 
+                bm25_threshold=self.filter_threshold
+            )
+        else:
+            return PruningContentFilter(
+                threshold=self.filter_threshold, 
+                threshold_type="fixed", 
+                min_word_threshold=self.word_count_threshold
+            )
+
+    async def crawl_urls(self, urls: List[str], user_query: Optional[str] = None):
+        """
+        Crawl multiple URLs with content filtering
+        
+        Args:
+            urls (List[str]): List of URLs to crawl
+            user_query (Optional[str]): Query used for BM25 content filtering
+        
+        Returns:
+            List of crawl results
+        """
+        async with AsyncWebCrawler(
+            browser_type="chromium", 
+            headless=True, 
+            verbose=True
+        ) as crawler:
+            # Create appropriate content filter
+            content_filter = self._create_content_filter(user_query)
+            
+            # Run crawling for multiple URLs
+            results = await crawler.arun_many(
+                urls=urls,
+                word_count_threshold=self.word_count_threshold,
+                bypass_cache=True,
+                markdown_generator=DefaultMarkdownGenerator(
+                    content_filter=content_filter
+                ),
+                cache_mode=CacheMode.DISABLED,
+                exclude_external_links=True,
+                remove_overlay_elements=True,
+                simulate_user=True,
+                magic=True
+            )
+            
+            # Process and return crawl results
+            processed_results = []
+            for result in results:
+                crawl_result = {
+                    "url": result.url,
+                    "success": result.success,
+                    "title": result.metadata.get('title', 'N/A'),
+                    "content": result.markdown_v2.raw_markdown if result.success else result.error_message,
+                    "word_count": len(result.markdown_v2.raw_markdown.split()) if result.success else 0,
+                    "links": {
+                        "internal": len(result.links.get('internal', [])),
+                        "external": len(result.links.get('external', []))
+                    },
+                    "images": len(result.media.get('images', []))
+                }
+                processed_results.append(crawl_result)
+            
+            return processed_results
+
+    async def search_and_crawl(self, query: str) -> List[Dict]:
+        """
+        Perform web search and crawl the results
+        
+        Args:
+            query (str): Search query
+        
+        Returns:
+            List of crawled content results
+        """
+        # Perform web search
+        search_tool = self._create_web_search_tool()
+        try:
+            search_results = search_tool.invoke({"query": query})
+            
+            # Extract URLs from search results
+            urls = [result['link'] for result in search_results]
+            print(f"Found {len(urls)} URLs for query: {query}")
+            
+            # Crawl URLs
+            crawl_results = await self.crawl_urls(urls, user_query=query)
+            
+            return crawl_results
+        
+        except Exception as e:
+            print(f"Web search and crawl error: {e}")
+            return []
+
+def main():
+    # Example usage
+    crawler = AdvancedWebCrawler(
+        max_search_results=5,
+        word_count_threshold=50,
+        content_filter_type='f',
+        filter_threshold=0.48
+    )
+    
+    test_queries = [
+        "Latest developments in AI agents",
+        "Today's weather forecast in Kolkata",
+    ]
+    
+    for query in test_queries:
+        # Run search and crawl asynchronously
+        results = asyncio.run(crawler.search_and_crawl(query))
+        
+        print(f"\nResults for query: {query}")
+        for result in results:
+            print(f"URL: {result['url']}")
+            print(f"Success: {result['success']}")
+            print(f"Title: {result['title']}")
+            print(f"Word Count: {result['word_count']}")
+            print(f"Content Preview: {result['content'][:500]}...\n")
+
+if __name__ == "__main__":
+    main()

src/vectorstore/pinecone_db.py

@@ -0,0 +1,282 @@
+from src.data_processing.loader import MultiFormatDocumentLoader
+from src.data_processing.chunker import SDPMChunker, BGEM3Embeddings
+
+import pandas as pd
+from typing import List, Dict, Any
+from pinecone import Pinecone, ServerlessSpec
+import time
+from tqdm import tqdm
+from dotenv import load_dotenv
+import os
+
+
+load_dotenv()
+
+# API Keys
+PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
+
+embedding_model = BGEM3Embeddings(model_name="BAAI/bge-m3")
+
+
+def load_documents(file_paths: List[str], output_path='./data/output.md'):
+    """
+    Load documents from multiple sources and combine them into a single markdown file
+    """
+    loader = MultiFormatDocumentLoader(
+        file_paths=file_paths,
+        enable_ocr=False,
+        enable_tables=True
+    )
+    
+    # Append all documents to the markdown file
+    with open(output_path, 'w') as f:
+        for doc in loader.lazy_load():
+            # Add metadata as YAML frontmatter
+            f.write('---\n')
+            for key, value in doc.metadata.items():
+                f.write(f'{key}: {value}\n')
+            f.write('---\n\n')
+            f.write(doc.page_content)
+            f.write('\n\n')
+    
+    return output_path
+
+def process_chunks(markdown_path: str, chunk_size: int = 256, 
+                  threshold: float = 0.7, skip_window: int = 2):
+    """
+    Process the markdown file into chunks and prepare for vector storage
+    """
+    chunker = SDPMChunker(
+        embedding_model=embedding_model,
+        chunk_size=chunk_size,
+        threshold=threshold,
+        skip_window=skip_window
+    )
+    
+    # Read the markdown file
+    with open(markdown_path, 'r') as file:
+        text = file.read()
+    
+    # Generate chunks
+    chunks = chunker.chunk(text)
+    
+    # Prepare data for Parquet
+    processed_chunks = []
+    for chunk in chunks:
+        
+        processed_chunks.append({
+            'text': chunk.text,
+            'token_count': chunk.token_count,
+            'start_index': chunk.start_index,
+            'end_index': chunk.end_index,
+            'num_sentences': len(chunk.sentences),
+        })
+    
+    return processed_chunks
+
+def save_to_parquet(chunks: List[Dict[str, Any]], output_path='./data/chunks.parquet'):
+    """
+    Save processed chunks to a Parquet file
+    """
+    df = pd.DataFrame(chunks)
+    print(f"Saving to Parquet: {output_path}")
+    df.to_parquet(output_path)
+    print(f"Saved to Parquet: {output_path}")
+    return output_path
+
+
+class PineconeRetriever:
+    def __init__(
+        self,
+        pinecone_client: Pinecone,
+        index_name: str,
+        namespace: str,
+        embedding_generator: BGEM3Embeddings
+    ):
+        """Initialize the retriever with Pinecone client and embedding generator.
+        
+        Args:
+            pinecone_client: Initialized Pinecone client
+            index_name: Name of the Pinecone index
+            namespace: Namespace in the index
+            embedding_generator: BGEM3Embeddings instance
+        """
+        self.pinecone = pinecone_client
+        self.index = self.pinecone.Index(index_name)
+        self.namespace = namespace
+        self.embedding_generator = embedding_generator
+    
+    def invoke(self, question: str, top_k: int = 5):
+        """Retrieve similar documents for a question.
+        
+        Args:
+            question: Query string
+            top_k: Number of results to return
+            
+        Returns:
+            List of dictionaries containing retrieved documents
+        """
+        # Generate embedding for the question
+        question_embedding = self.embedding_generator.embed(question)
+        question_embedding = question_embedding.tolist()
+        # Query Pinecone
+        results = self.index.query(
+            namespace=self.namespace,
+            vector=question_embedding,
+            top_k=top_k,
+            include_values=False,
+            include_metadata=True
+        )
+        
+        # Format results
+        retrieved_docs = [
+            {"page_content": match.metadata["text"], "score": match.score} 
+            for match in results.matches
+        ]
+        
+        return retrieved_docs
+
+def ingest_data(
+    pc,
+    parquet_path: str,
+    text_column: str,
+    pinecone_client: Pinecone,
+    index_name= "vector-index",
+    namespace= "rag",
+    batch_size: int = 100
+):
+    """Ingest data from a Parquet file into Pinecone.
+    
+    Args:
+        parquet_path: Path to the Parquet file
+        text_column: Name of the column containing text data
+        pinecone_client: Initialized Pinecone client
+        index_name: Name of the Pinecone index
+        namespace: Namespace in the index
+        batch_size: Batch size for processing
+    """
+    # Read Parquet file
+    print(f"Reading Parquet file: {parquet_path}")
+    df = pd.read_parquet(parquet_path)
+    print(f"Total records: {len(df)}")
+    # Create or get index
+    if not pinecone_client.has_index(index_name):
+        pinecone_client.create_index(
+            name=index_name,
+            dimension=1024,  # BGE-M3 dimension
+            metric="cosine",
+            spec=ServerlessSpec(
+                cloud='aws',
+                region='us-east-1'
+            )
+        )
+        
+        # Wait for index to be ready
+        while not pinecone_client.describe_index(index_name).status['ready']:
+            time.sleep(1)
+    
+    index = pinecone_client.Index(index_name)
+    
+    # Process in batches
+    for i in tqdm(range(0, len(df), batch_size)):
+        batch_df = df.iloc[i:i+batch_size]
+        
+        # Generate embeddings for batch
+        texts = batch_df[text_column].tolist()
+        embeddings = embedding_model.embed_batch(texts)
+        print(f"embeddings for batch: {i}")
+        # Prepare records for upsert
+        records = []
+        for idx, (_, row) in enumerate(batch_df.iterrows()):
+            records.append({
+                "id": str(row.name),  # Using DataFrame index as ID
+                "values": embeddings[idx],
+                "metadata": {"text": row[text_column]}
+            })
+        
+        # Upsert to Pinecone
+        index.upsert(vectors=records, namespace=namespace)
+        
+        # Small delay to handle rate limits
+        time.sleep(0.5)
+
+def get_retriever(
+    pinecone_client: Pinecone,
+    index_name= "vector-index",
+    namespace= "rag"
+):
+    """Create and return a PineconeRetriever instance.
+    
+    Args:
+        pinecone_client: Initialized Pinecone client
+        index_name: Name of the Pinecone index
+        namespace: Namespace in the index
+        
+    Returns:
+        Configured PineconeRetriever instance
+    """
+    return PineconeRetriever(
+        pinecone_client=pinecone_client,
+        index_name=index_name,
+        namespace=namespace,
+        embedding_generator=embedding_model
+    )
+    
+def main():
+    # Initialize Pinecone client
+    pc = Pinecone(api_key=PINECONE_API_KEY)
+    
+    # Define input files
+    file_paths=[
+        # './data/2404.19756v1.pdf',
+        # './data/OD429347375590223100.pdf',
+        # './data/Project Report Format.docx',
+        './data/UNIT 2 GENDER BASED VIOLENCE.pptx'
+    ]
+
+    # Process pipeline
+    try:
+        # Step 1: Load and combine documents
+        # print("Loading documents...")
+        # markdown_path = load_documents(file_paths)
+        
+        # # Step 2: Process into chunks with embeddings
+        # print("Processing chunks...")
+        # chunks = process_chunks(markdown_path)
+        
+        # # Step 3: Save to Parquet
+        # print("Saving to Parquet...")
+        # parquet_path = save_to_parquet(chunks)
+        
+        # # Step 4: Ingest into Pinecone
+        # print("Ingesting into Pinecone...")
+        # ingest_data(
+            # pc,
+        #     parquet_path=parquet_path,
+        #     text_column="text",
+        #     pinecone_client=pc,
+        # )
+        
+        # Step 5: Test retrieval
+        print("\nTesting retrieval...")
+        retriever = get_retriever(
+            pinecone_client=pc,
+            index_name="vector-index",
+            namespace="rag"
+        )
+        
+        results = retriever.invoke(
+            question="describe the gender based violence",
+            top_k=5
+        )
+        
+        for i, doc in enumerate(results, 1):
+            print(f"\nResult {i}:")
+            print(f"Content: {doc['page_content']}...")
+            print(f"Score: {doc['score']}")
+            
+    except Exception as e:
+        print(f"Error in pipeline: {str(e)}")
+
+if __name__ == "__main__":
+    main()