DOne some testing and fixed the retrieving context by date.

setup.py

@@ -0,0 +1,53 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="chatbot",
+    version="1.0.0",
+    packages=find_packages(),
+    install_requires=[
+        # Web Framework
+        "fastapi",
+        "uvicorn",
+        
+        # AI/ML
+        "torch",
+        "transformers",
+        "sentence-transformers",
+        "huggingface_hub",
+        
+        # LLM Providers
+        "openai",
+        "anthropic",
+        "ollama",
+        
+        # Data Validation & Processing
+        "pydantic",
+        "email-validator",
+        "numpy",
+        "pandas",
+        
+        # Database & Storage
+        "pymongo",
+        "motor",
+        "chromadb",
+        "aiosqlite",
+        
+        # Document Processing
+        "PyPDF2",
+        "python-docx",
+        "python-magic-bin==0.4.14",
+        "openpyxl",
+        "xlrd",
+        "striprtf",
+        "beautifulsoup4",
+        
+        # Utilities
+        "python-dotenv",
+        "requests",
+        "tiktoken",
+        "psutil",
+        
+        # Google Integration
+        "google-auth-oauthlib==0.4.6"
+    ]
+)

src/agents/enhanced_context_manager.py

@@ -0,0 +1,202 @@
+from typing import List, Dict, Optional, Tuple
+import spacy
+from collections import defaultdict
+
+class EnhancedContextManager:
+    def __init__(self):
+        """Initialize the context manager with NLP components"""
+        # Load spaCy model for NER and dependency parsing
+        self.nlp = spacy.load("en_core_web_sm")
+        # Track entities and their mentions across conversation
+        self.entity_mentions = defaultdict(list)
+        # Track conversation turns
+        self.conversation_turns = []
+        # Track last processed entity
+        self.last_entity = None
+        # Track last full response context
+        self.last_full_context = None
+        
+    def process_turn(self, query: str, response: str) -> None:
+        """Process a conversation turn to extract and track entities"""
+        # Parse query and response
+        query_doc = self.nlp(query)
+        response_doc = self.nlp(response)
+        
+        # Extract and track entities from both query and response
+        turn_entities = self._extract_entities(query_doc, response_doc)
+        
+        # Store the turn with its entities
+        self.conversation_turns.append({
+            'query': query,
+            'response': response,
+            'entities': turn_entities
+        })
+        
+        # Update entity mentions
+        for entity, info in turn_entities.items():
+            self.entity_mentions[entity].append({
+                'turn_index': len(self.conversation_turns) - 1,
+                'info': info
+            })
+            
+        # Update last entity and full context
+        if turn_entities:
+            # Prioritize entities in response, then query
+            primary_entity = (
+                list(turn_entities.keys())[0] if turn_entities 
+                else None
+            )
+            self.last_entity = primary_entity
+        
+        # Store full context for potential reference
+        self.last_full_context = f"{query} {response}"
+
+    def _extract_entities(self, query_doc, response_doc) -> Dict:
+        """Extract named entities and their properties"""
+        entities = {}
+        
+        # Process both query and response documents
+        for doc in [query_doc, response_doc]:
+            for ent in doc.ents:
+                # Store entity with its type and text
+                entities[ent.text] = {
+                    'type': ent.label_,
+                    'text': ent.text,
+                    'mentions': [tok.text for tok in doc if tok.head == ent.root]
+                }
+                
+        return entities
+
+    def resolve_pronouns(self, current_query: str) -> Optional[str]:
+        """
+        Resolve pronouns in the current query based on conversation history
+        
+        Args:
+            current_query (str): Current query with potential pronouns
+        
+        Returns:
+            Optional[str]: Query with resolved pronouns, or None if no resolution needed
+        """
+        if not self.conversation_turns:
+            return None
+            
+        query_doc = self.nlp(current_query)
+        
+        # Find pronouns in current query
+        pronouns = [token for token in query_doc if token.pos_ == "PRON"]
+        if not pronouns:
+            return None
+        
+        for pronoun in pronouns:
+            replacement = self._find_antecedent(pronoun.text)
+            if replacement:
+                # Replace the pronoun with the most likely antecedent
+                new_query = current_query.replace(pronoun.text, replacement)
+                return new_query
+                
+        return None
+
+    def _find_antecedent(self, pronoun: str) -> Optional[str]:
+        """
+        Find the most recent matching entity for a pronoun
+        
+        Args:
+            pronoun (str): Pronoun to resolve
+        
+        Returns:
+            Optional[str]: Resolved entity or None
+        """
+        # Pronoun to gender/number mapping
+        pronoun_properties = {
+            'he': {'gender': 'male', 'number': 'singular'},
+            'she': {'gender': 'female', 'number': 'singular'},
+            'they': {'gender': None, 'number': 'plural'},
+            'his': {'gender': 'male', 'number': 'singular'},
+            'her': {'gender': 'female', 'number': 'singular'},
+            'their': {'gender': None, 'number': 'plural'}
+        }
+        
+        # Normalize pronoun
+        pronoun_lower = pronoun.lower().rstrip('s')
+        
+        # If not a known pronoun, return None
+        if pronoun_lower not in pronoun_properties:
+            return None
+        
+        # If a named entity was recently mentioned, use it first
+        if self.last_entity:
+            return self.last_entity
+        
+        # Fallback to last full context if no specific entity found
+        if self.last_full_context:
+            return self.last_full_context.split()[0]
+        
+        return None
+
+    def enhance_query(self, current_query: str) -> str:
+        """
+        Enhance current query with context and resolved pronouns
+        
+        Args:
+            current_query (str): Original query
+        
+        Returns:
+            str: Enhanced query with additional context
+        """
+        # First try to resolve pronouns
+        resolved_query = self.resolve_pronouns(current_query)
+        
+        # If pronouns are resolved, use the resolved query
+        if resolved_query:
+            return resolved_query
+        
+        # Get relevant context
+        context = self._get_relevant_context(current_query)
+        
+        # If context found, prepend it to the query
+        if context:
+            return f"{context} {current_query}"
+        
+        # If no context resolution, return original query
+        return current_query
+
+    def _get_relevant_context(self, query: str) -> Optional[str]:
+        """
+        Get relevant context from conversation history
+        
+        Args:
+            query (str): Current query
+        
+        Returns:
+            Optional[str]: Relevant context or None
+        """
+        if not self.conversation_turns:
+            return None
+        
+        # Get the most recent turn
+        recent_turn = self.conversation_turns[-1]
+        
+        # If the current query contains a pronoun and we have last full context
+        if any(token.pos_ == "PRON" for token in self.nlp(query)):
+            return self.last_full_context
+        
+        return None
+
+    def get_conversation_context(self) -> List[Dict]:
+        """Get processed conversation context"""
+        return self.conversation_turns
+
+    def record_last_context(self, last_context: Optional[str] = None) -> None:
+        """
+        Manually record last context if needed
+        
+        Args:
+            last_context (Optional[str]): Last context to manually set
+        """
+        if last_context:
+            self.last_full_context = last_context
+            # Try to extract an entity from the context
+            doc = self.nlp(last_context)
+            entities = [ent.text for ent in doc.ents]
+            if entities:
+                self.last_entity = entities[0]

src/agents/rag_agent.py

@@ -1,8 +1,8 @@
-# src/agents/rag_agent.py
-from typing import List, Optional, Tuple, Dict
+from typing import List, Dict, Optional, Tuple
 import uuid
 
 from .excel_aware_rag import ExcelAwareRAGAgent
+from .enhanced_context_manager import EnhancedContextManager
 from ..llms.base_llm import BaseLLM
 from src.embeddings.base_embedding import BaseEmbedding
 from src.vectorstores.base_vectorstore import BaseVectorStore
@@ -21,17 +21,7 @@ class RAGAgent(ExcelAwareRAGAgent):
         max_history_tokens: int = 4000,
         max_history_messages: int = 10
     ):
-        """
-        Initialize RAG Agent
-        
-        Args:
-            llm (BaseLLM): Language model instance
-            embedding (BaseEmbedding): Embedding model instance
-            vector_store (BaseVectorStore): Vector store instance
-            mongodb (MongoDBStore): MongoDB store instance
-            max_history_tokens (int): Maximum tokens in conversation history
-            max_history_messages (int): Maximum messages to keep in history
-        """
+        """Initialize RAG Agent with enhanced context management"""
         super().__init__()  # Initialize ExcelAwareRAGAgent
         self.llm = llm
         self.embedding = embedding
@@ -41,6 +31,9 @@ class RAGAgent(ExcelAwareRAGAgent):
             max_tokens=max_history_tokens,
             max_messages=max_history_messages
         )
+        # Add enhanced context management while preserving existing functionality
+        self.context_manager = EnhancedContextManager()
+        logger.info("RAGAgent initialized with enhanced context management")
 
     async def generate_response(
         self,
@@ -48,11 +41,38 @@ class RAGAgent(ExcelAwareRAGAgent):
         conversation_id: Optional[str],
         temperature: float,
         max_tokens: Optional[int] = None,
-        context_docs: Optional[List[str]] = None
+        context_docs: Optional[List[str]] = None,
+        stream: bool = False,
+        custom_roles: Optional[List[Dict[str, str]]] = None
     ) -> RAGResponse:
-        """Generate response with specific handling for different query types"""
+        """
+        Generate a response with comprehensive context and role management
+    
+        Args:
+            query (str): User query
+            conversation_id (Optional[str]): Conversation identifier
+            temperature (float): LLM temperature for response generation
+            max_tokens (Optional[int]): Maximum tokens for response
+            context_docs (Optional[List[str]]): Pre-retrieved context documents
+            stream (bool): Whether to stream the response
+            custom_roles (Optional[List[Dict[str, str]]]): Custom role instructions
+    
+        Returns:
+            RAGResponse: Generated response with context and metadata
+        """
         try:
-            # First, check if this is an introduction/welcome message query
+            logger.info(f"Generating response for query: {query}")
+        
+            # Apply custom roles if provided
+            if custom_roles:
+                for role in custom_roles:
+                    # Modify query or context based on role
+                    if role.get('name') == 'introduction_specialist':
+                        query += " Provide a concise, welcoming response."
+                    elif role.get('name') == 'knowledge_based_specialist':
+                        query += " Ensure response is precise and directly from available knowledge."
+
+            # Introduction Handling
             is_introduction = (
                 "wants support" in query and 
                 "This is Introduction" in query and 
@@ -60,7 +80,7 @@ class RAGAgent(ExcelAwareRAGAgent):
             )
 
             if is_introduction:
-                # Handle introduction message - no context needed
+                logger.info("Processing introduction message")
                 welcome_message = self._handle_contact_query(query)
                 return RAGResponse(
                     response=welcome_message,
@@ -69,69 +89,118 @@ class RAGAgent(ExcelAwareRAGAgent):
                     scores=None
                 )
 
-            # Get conversation history if conversation_id exists
+            # Conversation History Processing
             history = []
+            last_context = None
             if conversation_id:
+                logger.info(f"Retrieving conversation history for ID: {conversation_id}")
                 history = await self.mongodb.get_recent_messages(
                     conversation_id,
                     limit=self.conversation_manager.max_messages
                 )
             
-                # Get relevant history within token limits
+                # Process history for conversation manager
                 history = self.conversation_manager.get_relevant_history(
                     messages=history,
                     current_query=query
                 )
+            
+                # Process in enhanced context manager
+                for msg in history:
+                    self.context_manager.process_turn(
+                        msg.get('query', ''),
+                        msg.get('response', '')
+                    )
+            
+                # Get last context if available
+                if history and history[-1].get('response'):
+                    last_context = history[-1]['response']
+
+            # Query Enhancement
+            enhanced_query = self.context_manager.enhance_query(query)
+        
+            # Manual Pronoun Handling Fallback
+            if enhanced_query == query:
+                pronoun_map = {
+                    'his': 'he',
+                    'her': 'she',
+                    'their': 'they'
+                }
+                words = query.lower().split()
+                for pronoun, replacement in pronoun_map.items():
+                    if pronoun in words:
+                        # Try to use last context
+                        if last_context:
+                            self.context_manager.record_last_context(last_context)
+                            enhanced_query = self.context_manager.enhance_query(query)
+                            break
 
-            # Retrieve context if not provided
+            logger.info(f"Enhanced query: {enhanced_query}")
+
+            # Context Retrieval
             if not context_docs:
+                logger.info("Retrieving context for enhanced query")
                 context_docs, sources, scores = await self.retrieve_context(
-                    query=query,
+                    enhanced_query,
                     conversation_history=history
                 )
             else:
-                sources = None
+                sources = []
                 scores = None
 
-            # Check if we have any relevant context
+            # Context Fallback Mechanism
             if not context_docs:
-                return RAGResponse(
-                    response="Information about this is not available, do you want to inquire about something else?",
-                    context_docs=[],
-                    sources=[],
-                    scores=None
-                )
+                # If no context and last context exists, use it
+                if last_context:
+                    context_docs = [last_context]
+                    sources = [{"source": "previous_context"}]
+                    scores = [1.0]
+                else:
+                    logger.info("No relevant context found")
+                    return RAGResponse(
+                        response="Information about this is not available, do you want to inquire about something else?",
+                        context_docs=[],
+                        sources=[],
+                        scores=None
+                    )
 
-            # Check if this is an Excel-related query
+            # Excel-specific Content Handling
             has_excel_content = any('Sheet:' in doc for doc in context_docs)
             if has_excel_content:
+                logger.info("Processing Excel-specific content")
                 try:
-                    context_docs = self._process_excel_context(context_docs, query)
+                    context_docs = self._process_excel_context(context_docs, enhanced_query)
                 except Exception as e:
                     logger.warning(f"Error processing Excel context: {str(e)}")
 
-            # Generate prompt with context and history
-            augmented_prompt = self.conversation_manager.generate_prompt_with_history(
-                current_query=query,
+            # Prompt Generation with Conversation History
+            prompt = self.conversation_manager.generate_prompt_with_history(
+                current_query=enhanced_query,
                 history=history,
                 context_docs=context_docs
             )
 
-            # Generate initial response
+            # Streaming Response Generation
+            if stream:
+                # TODO: Implement actual streaming logic
+                # This is a placeholder and needs proper implementation
+                logger.warning("Streaming not fully implemented")
+
+            # Standard Response Generation
             response = self.llm.generate(
-                prompt=augmented_prompt,
+                prompt=prompt,
                 temperature=temperature,
                 max_tokens=max_tokens
             )
 
-            # Clean the response
+            # Response Cleaning
             cleaned_response = self._clean_response(response)
-
-            # For Excel queries, enhance the response
+        
+            # Excel Response Enhancement
             if has_excel_content:
                 try:
                     enhanced_response = await self.enhance_excel_response(
-                        query=query,
+                        query=enhanced_query,
                         response=cleaned_response,
                         context_docs=context_docs
                     )
@@ -140,61 +209,30 @@ class RAGAgent(ExcelAwareRAGAgent):
                 except Exception as e:
                     logger.warning(f"Error enhancing Excel response: {str(e)}")
 
-            # Return the final response
+            # Context Tracking
+            self.context_manager.process_turn(query, cleaned_response)
+
+            # Metadata Generation
+            metadata = {
+                'llm_provider': getattr(self.llm, 'model_name', 'unknown'),
+                'temperature': temperature,
+                'conversation_id': conversation_id,
+                'context_sources': sources,
+                'has_excel_content': has_excel_content
+            }
+
+            logger.info("Successfully generated response")
             return RAGResponse(
                 response=cleaned_response,
                 context_docs=context_docs,
                 sources=sources,
-                scores=scores
+                scores=scores,
+                metadata=metadata  # Added metadata
             )
 
         except Exception as e:
-            logger.error(f"Error in SystemInstructionsRAGAgent: {str(e)}")
+            logger.error(f"Error in generate_response: {str(e)}")
             raise
-    
-    def _create_response_prompt(self, query: str, context_docs: List[str]) -> str:
-        """
-        Create prompt for generating response from context
-    
-        Args:
-            query (str): User query
-            context_docs (List[str]): Retrieved context documents
-        
-        Returns:
-            str: Formatted prompt for the LLM
-        """
-        if not context_docs:
-            return f"Query: {query}\nResponse: Information about this is not available, do you want to inquire about something else?"
-
-        # Format context documents
-        formatted_context = "\n\n".join(
-            f"Context {i+1}:\n{doc.strip()}"
-            for i, doc in enumerate(context_docs)
-            if doc and doc.strip()
-        )
-
-        # Build the prompt with detailed instructions
-        prompt = f"""You are a knowledgeable assistant. Use the following context to answer the query accurately and informatively.
-
-    Context Information:
-    {formatted_context}
-
-    Query: {query}
-
-    Instructions:
-    1. Base your response ONLY on the information provided in the context above
-    2. If the context contains numbers, statistics, or specific details, include them in your response
-    3. Keep your response focused and relevant to the query
-    4. Use clear and professional language
-    5. If the context includes technical terms, explain them appropriately
-    6. Do not make assumptions or add information not present in the context
-    7. If specific sections of a report are mentioned, maintain their original structure
-    8. Format the response in a clear, readable manner
-    9. If the context includes chronological information, maintain the proper sequence
-
-    Response:"""
-
-        return prompt
 
     async def retrieve_context(
         self,
@@ -202,59 +240,127 @@ class RAGAgent(ExcelAwareRAGAgent):
         conversation_history: Optional[List[Dict]] = None,
         top_k: int = 3
     ) -> Tuple[List[str], List[Dict], Optional[List[float]]]:
-        """
-        Retrieve context with conversation history enhancement
-        """
-        # Enhance query with conversation history
-        if conversation_history:
-            recent_queries = [
-                msg['query'] for msg in conversation_history[-2:]
-                if msg.get('query')
-            ]
-            enhanced_query = " ".join([*recent_queries, query])
-        else:
-            enhanced_query = query
-
-        # Debug log the enhanced query
-        logger.info(f"Enhanced query: {enhanced_query}")
-
-        # Embed the enhanced query
-        query_embedding = self.embedding.embed_query(enhanced_query)
+        """Retrieve context with both original and enhanced handling"""
+        try:
+            logger.info(f"Retrieving context for query: {query}")
+            
+            # Enhance query using both managers
+            if conversation_history:
+                # Get the last two messages for immediate context
+                recent_messages = conversation_history[-2:]
+                
+                # Extract queries and responses for context
+                context_parts = []
+                for msg in recent_messages:
+                    if msg.get('query'):
+                        context_parts.append(msg['query'])
+                    if msg.get('response'):
+                        response = msg['response']
+                        if "Information about this is not available" not in response:
+                            context_parts.append(response)
+
+                # Combine with current query
+                enhanced_query = f"{' '.join(context_parts)} {query}".strip()
+                logger.info(f"Enhanced query with history: {enhanced_query}")
+            else:
+                enhanced_query = query
+
+            # Debug log the enhanced query
+            logger.info(f"Final enhanced query: {enhanced_query}")
+
+            # Embed the enhanced query
+            query_embedding = self.embedding.embed_query(enhanced_query)
     
-        # Debug log embedding shape
-        logger.info(f"Query embedding shape: {len(query_embedding)}")
+            # Debug log embedding shape
+            logger.info(f"Query embedding shape: {len(query_embedding)}")
 
-        # Retrieve similar documents
-        results = self.vector_store.similarity_search(
-            query_embedding,
-            top_k=top_k
-        )
+            # Retrieve similar documents
+            results = self.vector_store.similarity_search(
+                query_embedding,
+                top_k=top_k
+            )
+
+            # Debug log search results
+            logger.info(f"Number of search results: {len(results)}")
+            for i, result in enumerate(results):
+                logger.info(f"Result {i} score: {result.get('score', 'N/A')}")
+                logger.info(f"Result {i} text preview: {result.get('text', '')[:100]}...")
+
+            if not results:
+                logger.info("No results found in similarity search")
+                return [], [], None
+
+            # Process results
+            documents = [doc['text'] for doc in results]
+            sources = [self._convert_metadata_to_strings(doc['metadata']) 
+                      for doc in results]
+            scores = [doc['score'] for doc in results 
+                     if doc.get('score') is not None]
 
-        # Debug log search results
-        logger.info(f"Number of search results: {len(results)}")
-        for i, result in enumerate(results):
-            logger.info(f"Result {i} score: {result.get('score', 'N/A')}")
-            logger.info(f"Result {i} text preview: {result.get('text', '')[:100]}...")
+            # Return scores only if available for all documents
+            if len(scores) != len(documents):
+                scores = None
+
+            logger.info(f"Retrieved {len(documents)} relevant documents")
+            return documents, sources, scores
+
+        except Exception as e:
+            logger.error(f"Error in retrieve_context: {str(e)}")
+            raise
 
-        # Process results
-        documents = [doc['text'] for doc in results]
-        sources = [self._convert_metadata_to_strings(doc['metadata']) 
-                  for doc in results]
-        scores = [doc['score'] for doc in results 
-                 if doc.get('score') is not None]
+    def _clean_response(self, response: str) -> str:
+        """Clean response text while preserving key information"""
+        if not response:
+            return response
 
-        # Return scores only if available for all documents
-        if len(scores) != len(documents):
-            scores = None
+        # Keep only the most common phrases to remove
+        phrases_to_remove = [
+            "Based on the context,",
+            "According to the documents,",
+            "From the information available,",
+            "Based on the provided information,",
+            "I apologize,"
+        ]
+    
+        cleaned_response = response
+        for phrase in phrases_to_remove:
+            cleaned_response = cleaned_response.replace(phrase, "").strip()
+    
+        cleaned_response = " ".join(cleaned_response.split())
 
-        return documents, sources, scores
+        if not cleaned_response:
+            return response
+    
+        if cleaned_response[0].islower():
+            cleaned_response = cleaned_response[0].upper() + cleaned_response[1:]
+    
+        return cleaned_response
 
     def _convert_metadata_to_strings(self, metadata: Dict) -> Dict:
-        """Convert numeric metadata values to strings"""
-        converted = {}
-        for key, value in metadata.items():
-            if isinstance(value, (int, float)):
-                converted[key] = str(value)
-            else:
-                converted[key] = value
-        return converted
+        """Convert metadata values to strings"""
+        try:
+            return {
+                key: str(value) if isinstance(value, (int, float)) else value
+                for key, value in metadata.items()
+            }
+        except Exception as e:
+            logger.error(f"Error converting metadata: {str(e)}")
+            return metadata
+
+    def _handle_contact_query(self, query: str) -> str:
+        """Handle contact/introduction queries"""
+        try:
+            name_start = query.find('name: "') + 7
+            name_end = query.find('"', name_start)
+            name = query[name_start:name_end] if name_start > 6 and name_end != -1 else "there"
+
+            is_returning = (
+                "An old user with name:" in query and 
+                "wants support again" in query
+            )
+            
+            return f"Welcome back {name}, How can I help you?" if is_returning else f"Welcome {name}, How can I help you?"
+
+        except Exception as e:
+            logger.error(f"Error handling contact query: {str(e)}")
+            return "Welcome, How can I help you?"

src/agents/rag_agent_manager.py

@@ -0,0 +1,77 @@
+# src/agents/rag_agent_manager.py
+from typing import Optional
+import weakref
+
+from src.agents.system_instructions_rag import SystemInstructionsRAGAgent
+from src.llms.base_llm import BaseLLM
+from src.embeddings.base_embedding import BaseEmbedding
+from src.vectorstores.base_vectorstore import BaseVectorStore
+from src.db.mongodb_store import MongoDBStore
+from src.utils.logger import logger
+
+class RAGAgentManager:
+    """
+    Singleton manager for RAG Agent instances with intelligent caching
+    """
+    _instance = None
+    
+    def __new__(cls):
+        if not cls._instance:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+    
+    def __init__(self):
+        # Ensure this is only initialized once
+        if not hasattr(self, '_initialized'):
+            self._rag_agent = None
+            self._initialized = True
+    
+    def get_rag_agent(
+        self,
+        llm: BaseLLM,
+        embedding_model: BaseEmbedding,
+        vector_store: BaseVectorStore,
+        mongodb: MongoDBStore
+    ) -> SystemInstructionsRAGAgent:
+        """
+        Get or create a singleton RAG agent instance with intelligent caching
+        
+        Args:
+            llm: Language Model instance
+            embedding_model: Embedding model instance
+            vector_store: Vector store instance
+            mongodb: MongoDB store instance
+        
+        Returns:
+            SystemInstructionsRAGAgent: Singleton instance of the RAG agent
+        """
+        # If RAG agent exists and all dependencies are the same, return it
+        if self._rag_agent is not None:
+            logger.info("Reusing existing RAG agent instance")
+            return self._rag_agent
+        
+        try:
+            logger.info("Creating new RAG agent instance")
+            # Create the agent
+            self._rag_agent = SystemInstructionsRAGAgent(
+                llm=llm,
+                embedding=embedding_model,
+                vector_store=vector_store,
+                mongodb=mongodb
+            )
+            
+            return self._rag_agent
+        
+        except Exception as e:
+            logger.error(f"Error creating RAG agent: {str(e)}")
+            raise
+    
+    def reset_rag_agent(self):
+        """
+        Reset the RAG agent instance
+        """
+        logger.info("Resetting RAG agent instance")
+        self._rag_agent = None
+
+# Create a global instance for easy import
+rag_agent_manager = RAGAgentManager()

src/agents/system_instructions_rag.py

@@ -1,33 +1,53 @@
 # src/agents/system_instructions_rag.py
-from typing import List, Dict, Optional
-from src.agents.rag_agent import RAGResponse
-from src.utils.logger import logger
+from typing import List, Dict, Optional, Tuple
+import spacy
 from src.agents.rag_agent import RAGAgent
+from src.llms.base_llm import BaseLLM
+from src.embeddings.base_embedding import BaseEmbedding
+from src.vectorstores.base_vectorstore import BaseVectorStore
+from src.db.mongodb_store import MongoDBStore
+from src.models.rag import RAGResponse
+from src.utils.logger import logger
 
 class SystemInstructionsRAGAgent(RAGAgent):
-    """RAG Agent with enhanced system instructions for specific use cases"""
-    
+    def __init__(
+        self,
+        llm: BaseLLM,
+        embedding: BaseEmbedding,
+        vector_store: BaseVectorStore,
+        mongodb: MongoDBStore,
+        max_history_tokens: int = 4000,
+        max_history_messages: int = 10
+    ):
+        """Initialize SystemInstructionsRAGAgent with enhanced context management"""
+        super().__init__(
+            llm=llm,
+            embedding=embedding,
+            vector_store=vector_store,
+            mongodb=mongodb,
+            max_history_tokens=max_history_tokens,
+            max_history_messages=max_history_messages
+        )
+        self.nlp = spacy.load("en_core_web_sm")
+
     async def generate_response(
         self,
         query: str,
         conversation_id: Optional[str] = None,
         temperature: float = 0.7,
         max_tokens: Optional[int] = None,
-        context_docs: Optional[List[str]] = None
+        context_docs: Optional[List[str]] = None,
+        stream: bool = False
     ) -> RAGResponse:
-        """
-        Generate response with specific handling for introduction and no-context cases
-        """
+        """Generate response with guaranteed context handling"""
         try:
-            # First, check if this is an introduction/welcome message query
-            is_introduction = (
-                "wants support" in query and 
-                "This is Introduction" in query and 
-                ("A new user with name:" in query or "An old user with name:" in query)
-            )
+            logger.info(f"Processing query: {query}")
+            
+            # Store original context if provided
+            original_context = context_docs
 
-            if is_introduction:
-                # Handle introduction message - no context needed
+            # Handle introduction queries
+            if self._is_introduction_query(query):
                 welcome_message = self._handle_contact_query(query)
                 return RAGResponse(
                     response=welcome_message,
@@ -36,107 +56,249 @@ class SystemInstructionsRAGAgent(RAGAgent):
                     scores=None
                 )
 
-            # For all other queries, proceed with context-based response
-            if not context_docs:
-                context_docs, sources, scores = await self.retrieve_context(
+            # Get and process conversation history
+            history = []
+            if conversation_id:
+                history = await self.mongodb.get_recent_messages(
+                    conversation_id,
+                    limit=self.conversation_manager.max_messages
+                )
+                
+                # Process history in context manager
+                for msg in history:
+                    if msg.get('query') and msg.get('response'):
+                        self.context_manager.process_turn(msg['query'], msg['response'])
+
+            # Initialize context tracking
+            current_context = None
+            sources = []
+            scores = None
+
+            # Multi-stage context retrieval
+            if original_context:
+                current_context = original_context
+            else:
+                # Try with original query first
+                current_context, sources, scores = await self.retrieve_context(
                     query,
-                    conversation_history=[]
+                    conversation_history=history
                 )
 
-            # Check if we have relevant context
-            has_relevant_context = self._check_context_relevance(query, context_docs or [])
+                # If no context, try with enhanced query
+                if not current_context:
+                    enhanced_query = self.context_manager.enhance_query(query)
+                    if enhanced_query != query:
+                        current_context, sources, scores = await self.retrieve_context(
+                            enhanced_query,
+                            conversation_history=history
+                        )
+
+                # If still no context, try history fallback
+                if not current_context:
+                    current_context, sources = self._get_context_from_history(history)
+                    
+                logger.info(f"Retrieved {len(current_context) if current_context else 0} context documents")
 
-            # If no relevant context found, return the standard message
+            # Check context relevance
+            has_relevant_context = self._check_context_relevance(query, current_context or [])
+            logger.info(f"Context relevance check result: {has_relevant_context}")
+
+            # Handle no context case
             if not has_relevant_context:
-                return RAGResponse(
-                    response="Information about this is not available, do you want to inquire about something else?",
-                    context_docs=[],
-                    sources=[],
-                    scores=None
-                )
+                return self._create_no_info_response()
 
-            # Generate response using context
-            prompt = self._create_response_prompt(query, context_docs)
+            # Generate response
+            prompt = self._create_response_prompt(query, current_context)
             response_text = self.llm.generate(
-                prompt,
+                prompt=prompt,
                 temperature=temperature,
                 max_tokens=max_tokens
             )
 
-            # Check if the generated response indicates no information
+            # Process and validate response
             cleaned_response = self._clean_response(response_text)
             if self._is_no_info_response(cleaned_response):
-                return RAGResponse(
-                    response="Information about this is not available, do you want to inquire about something else?",
-                    context_docs=[],
-                    sources=[],
-                    scores=None
-                )
+                return self._create_no_info_response()
+
+            # Update context tracking
+            self.context_manager.process_turn(query, cleaned_response)
+
+            # For Excel content, enhance the response
+            if any('Sheet:' in doc for doc in (current_context or [])):
+                try:
+                    cleaned_response = await self.enhance_excel_response(
+                        query=query,
+                        response=cleaned_response,
+                        context_docs=current_context
+                    )
+                except Exception as e:
+                    logger.warning(f"Error enhancing Excel response: {str(e)}")
 
             return RAGResponse(
                 response=cleaned_response,
-                context_docs=context_docs,
+                context_docs=current_context,
                 sources=sources,
                 scores=scores
             )
 
         except Exception as e:
-            logger.error(f"Error in SystemInstructionsRAGAgent: {str(e)}")
+            logger.error(f"Error in generate_response: {str(e)}")
             raise
 
-    def _is_no_info_response(self, response: str) -> bool:
-        """Check if the response indicates no information available"""
-        no_info_indicators = [
-            "i do not have",
-            "i don't have",
-            "no information",
-            "not available",
-            "could not find",
-            "couldn't find",
-            "cannot find"
-        ]
-        response_lower = response.lower()
-        return any(indicator in response_lower for indicator in no_info_indicators)
-
-    def _check_context_relevance(self, query: str, context_docs: List[str]) -> bool:
-        """Check if context contains information relevant to the query"""
-        if not context_docs:
-            return False
-
-        # Extract key terms from query (keeping important words)
-        query_words = query.lower().split()
-        stop_words = {'me', 'a', 'about', 'what', 'is', 'are', 'the', 'in', 'how', 'why', 'when', 'where'}
-    
-        # Remove only basic stop words, keep important terms like "report", "share", etc.
-        query_terms = {word for word in query_words if word not in stop_words}
-    
-        # Add additional relevant terms that might appear in the content
-        related_terms = {
-            'comprehensive',
-            'report',
-            'overview',
-            'summary',
-            'details',
-            'information'
+    def _convert_metadata_to_strings(self, metadata: Dict) -> Dict:
+        """Convert all metadata values to strings"""
+        return {
+            key: str(value) if value is not None else None
+            for key, value in metadata.items()
         }
-        query_terms.update(word for word in query_words if word in related_terms)
-    
-        # Check each context document for relevance
-        for doc in context_docs:
-            if not doc:
-                continue
-            doc_lower = doc.lower()
-        
-            # Consider document relevant if it contains any query terms
-            # or if it starts with common report headers
-            if any(term in doc_lower for term in query_terms) or \
-                any(header in doc_lower for header in ['overview', 'comprehensive report', 'summary']):
-                return True
+
+    async def retrieve_context(
+        self,
+        query: str,
+        conversation_history: Optional[List[Dict]] = None
+    ) -> Tuple[List[str], List[Dict], Optional[List[float]]]:
+        """Enhanced context retrieval with proper metadata type handling"""
+        try:
+            logger.info(f"Processing query for context retrieval: {query}")
             
-        return False
+            collection_data = self.vector_store.collection.get()
+            
+            if not collection_data or 'documents' not in collection_data:
+                logger.warning("No documents found in ChromaDB")
+                return [], [], None
+            
+            documents = collection_data['documents']
+            metadatas = collection_data.get('metadatas', [])
+            
+            # Clean and enhance query with date variations
+            clean_query = query.lower().strip()
+            
+            # Extract and enhance date information
+            import re
+            from datetime import datetime
+            
+            date_pattern = r'(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]* \d{1,2},? \d{4}'
+            dates = re.findall(date_pattern, clean_query.lower())
+            
+            enhanced_query = clean_query
+            target_date = None
+            
+            if dates:
+                try:
+                    date_obj = datetime.strptime(dates[0], '%b %d, %Y')
+                    target_date = date_obj.strftime('%b %d, %Y')
+                    
+                    date_variations = [
+                        date_obj.strftime('%B %d, %Y'),
+                        date_obj.strftime('%d/%m/%Y'),
+                        date_obj.strftime('%Y-%m-%d'),
+                        target_date
+                    ]
+                    
+                    enhanced_query = f"{clean_query} {' '.join(date_variations)}"
+                    
+                except ValueError as e:
+                    logger.warning(f"Error parsing date: {str(e)}")
+            
+            # First try exact date matching
+            exact_matches = []
+            exact_metadata = []
+            
+            if target_date:
+                for i, doc in enumerate(documents):
+                    if target_date in doc:
+                        logger.info(f"Found exact date match in document {i}")
+                        exact_matches.append(doc)
+                        if metadatas:
+                            # Convert metadata values to strings
+                            exact_metadata.append(self._convert_metadata_to_strings(metadatas[i]))
+                
+                if exact_matches:
+                    logger.info(f"Found {len(exact_matches)} exact date matches")
+                    document_id = exact_metadata[0].get('document_id') if exact_metadata else None
+                    
+                    if document_id:
+                        all_related_chunks = []
+                        all_related_metadata = []
+                        all_related_scores = []
+                        
+                        for i, doc in enumerate(documents):
+                            if metadatas[i].get('document_id') == document_id:
+                                all_related_chunks.append(doc)
+                                # Convert metadata values to strings
+                                all_related_metadata.append(self._convert_metadata_to_strings(metadatas[i]))
+                                all_related_scores.append(1.0)
+                        
+                        # Sort chunks by their index
+                        sorted_results = sorted(
+                            zip(all_related_chunks, all_related_metadata, all_related_scores),
+                            key=lambda x: int(x[1].get('chunk_index', '0'))  # Convert to int for sorting
+                        )
+                        
+                        sorted_chunks, sorted_metadata, sorted_scores = zip(*sorted_results)
+                        
+                        logger.info(f"Returning {len(sorted_chunks)} chunks from document {document_id}")
+                        return list(sorted_chunks), list(sorted_metadata), list(sorted_scores)
+            
+            # If no exact matches, use enhanced query for embedding search
+            logger.info("No exact matches found, using enhanced query for embedding search")
+            query_embedding = self.embedding.embed_query(enhanced_query)
+            
+            results = self.vector_store.similarity_search(
+                query_embedding,
+                top_k=5
+            )
+            
+            if not results:
+                logger.warning("No results found in similarity search")
+                return [], [], None
+            
+            context_docs = []
+            sources = []
+            scores = []
+            
+            sorted_results = sorted(results, key=lambda x: x.get('score', 0), reverse=True)
+            
+            for result in sorted_results:
+                score = result.get('score', 0)
+                if score > 0.3:
+                    context_docs.append(result.get('text', ''))
+                    # Convert metadata values to strings
+                    sources.append(self._convert_metadata_to_strings(result.get('metadata', {})))
+                    scores.append(score)
+            
+            if context_docs:
+                logger.info(f"Returning {len(context_docs)} documents from similarity search")
+                return context_docs, sources, scores
+            
+            logger.warning("No relevant documents found")
+            return [], [], None
+            
+        except Exception as e:
+            logger.error(f"Error in retrieve_context: {str(e)}")
+            logger.exception("Full traceback:")
+            return [], [], None
+        
+    def _is_introduction_query(self, query: str) -> bool:
+        """Check if query is an introduction message"""
+        return (
+            "wants support" in query and 
+            "This is Introduction" in query and 
+            ("A new user with name:" in query or "An old user with name:" in query)
+        )
+
+    def _get_context_from_history(
+        self,
+        history: List[Dict]
+    ) -> Tuple[Optional[List[str]], Optional[List[Dict]]]:
+        """Extract context from conversation history"""
+        for msg in reversed(history):
+            if msg.get('context') and not self._is_no_info_response(msg.get('response', '')):
+                return msg['context'], msg.get('sources', [])
+        return None, None
 
     def _create_response_prompt(self, query: str, context_docs: List[str]) -> str:
-        """Create prompt for generating response from context"""
+        """Create prompt for response generation"""
         formatted_context = '\n\n'.join(
             f"Context {i+1}:\n{doc.strip()}"
             for i, doc in enumerate(context_docs)
@@ -159,28 +321,17 @@ Instructions:
 Query: {query}
 Response:"""
 
-    def _handle_contact_query(self, query: str) -> str:
-        """Handle queries from /user/contact endpoint"""
-        try:
-            name_start = query.find('name: "') + 7
-            name_end = query.find('"', name_start)
-            name = query[name_start:name_end] if name_start > 6 and name_end != -1 else "there"
-
-            is_returning = (
-                "An old user with name:" in query and 
-                "wants support again" in query
-            )
-            
-            if is_returning:
-                return f"Welcome back {name}, How can I help you?"
-            return f"Welcome {name}, How can I help you?"
-
-        except Exception as e:
-            logger.error(f"Error handling contact query: {str(e)}")
-            return "Welcome, How can I help you?"
+    def _create_no_info_response(self) -> RAGResponse:
+        """Create standard response for no information case"""
+        return RAGResponse(
+            response="Information about this is not available, do you want to inquire about something else?",
+            context_docs=[],
+            sources=[],
+            scores=None
+        )
 
     def _clean_response(self, response: str) -> str:
-        """Clean response by removing unwanted phrases"""
+        """Clean response text"""
         if not response:
             return response
 
@@ -200,7 +351,6 @@ Response:"""
             "Here's what I found:",
             "Here's the information you requested:",
             "According to the provided information,",
-            "Based on the documents,",
             "The information suggests that",
             "From what I can see,",
             "Let me explain",
@@ -209,12 +359,6 @@ Response:"""
             "I can see that",
             "Sure,",
             "Well,",
-            "Based on the given context,",
-            "The available information shows that",
-            "From the context provided,",
-            "The documentation mentions that",
-            "According to the context,",
-            "As shown in the context,",
             "I apologize,"
         ]
         
@@ -230,4 +374,70 @@ Response:"""
         if cleaned_response[0].islower():
             cleaned_response = cleaned_response[0].upper() + cleaned_response[1:]
         
-        return cleaned_response
+        return cleaned_response
+
+    def _is_no_info_response(self, response: str) -> bool:
+        """Check if response indicates no information available"""
+        no_info_indicators = [
+            "i do not have",
+            "i don't have",
+            "no information",
+            "not available",
+            "could not find",
+            "couldn't find",
+            "cannot find",
+            "don't know",
+            "do not know",
+            "unable to find",
+            "no data",
+            "no relevant"
+        ]
+        response_lower = response.lower()
+        return any(indicator in response_lower for indicator in no_info_indicators)
+
+    def _check_context_relevance(self, query: str, context_docs: List[str]) -> bool:
+        """Enhanced context relevance checking"""
+        if not context_docs:
+            return False
+
+        # Clean and prepare query
+        clean_query = query.lower().strip()
+        query_terms = set(word for word in clean_query.split() 
+                         if word not in {'tell', 'me', 'about', 'what', 'is', 'the'})
+
+        for doc in context_docs:
+            if not doc:
+                continue
+
+            doc_lower = doc.lower()
+            
+            # For CSV-like content, check each line
+            lines = doc_lower.split('\n')
+            for line in lines:
+                # Check if any query term appears in the line
+                if any(term in line for term in query_terms):
+                    return True
+
+            # Also check the whole document for good measure
+            if any(term in doc_lower for term in query_terms):
+                return True
+
+        return False
+
+    def _handle_contact_query(self, query: str) -> str:
+        """Handle contact/introduction queries"""
+        try:
+            name_start = query.find('name: "') + 7
+            name_end = query.find('"', name_start)
+            name = query[name_start:name_end] if name_start > 6 and name_end != -1 else "there"
+
+            is_returning = (
+                "An old user with name:" in query and 
+                "wants support again" in query
+            )
+            
+            return f"Welcome back {name}, How can I help you?" if is_returning else f"Welcome {name}, How can I help you?"
+
+        except Exception as e:
+            logger.error(f"Error handling contact query: {str(e)}")
+            return "Welcome, How can I help you?"

src/implementations/document_service.py

@@ -105,21 +105,33 @@ class DocumentService:
         vector_store: ChromaVectorStore,
         background_tasks: BackgroundTasks
     ) -> DocumentResponse:
-        """Process a single file upload"""
-        # Generate UUID for document
+        """Process a single file upload with proper handle closure"""
         document_id = str(uuid4())
         filename = f"{document_id}_{file.filename}"
         file_path = self.permanent_dir / filename
         url_path = f"/docs/{filename}"
-        
+    
         try:
-            # Save file to permanent location
+            # Save file to permanent location using a context manager
             with open(file_path, "wb") as buffer:
                 shutil.copyfileobj(file.file, buffer)
 
-            # Process the document for vector store
-            processed_doc = await self.doc_processor.process_document(file_path)
+            # Close the uploaded file explicitly
+            await file.close()
+
+            # Process document with proper cleanup for Excel files
+            try:
+                processed_doc = await self.doc_processor.process_document(file_path)
             
+                # For Excel files, ensure pandas closes the file
+                if file_path.suffix.lower() in ['.xlsx', '.xls']:
+                    import gc
+                    gc.collect()  # Help cleanup any lingering file handles
+        
+            except Exception as proc_error:
+                logger.error(f"Error processing document: {str(proc_error)}")
+                raise
+
             # Store in MongoDB with url_path
             await self.mongodb.store_document(
                 document_id=document_id,
@@ -157,13 +169,13 @@ class DocumentService:
                     file_path.unlink()
                 except Exception as cleanup_error:
                     logger.error(f"Error cleaning up file {file_path}: {str(cleanup_error)}")
-            
+        
             # Clean up from MongoDB if document was created
             try:
                 await self.mongodb.delete_document(document_id)
             except Exception as db_cleanup_error:
                 logger.error(f"Error cleaning up MongoDB document {document_id}: {str(db_cleanup_error)}")
-            
+        
             logger.error(f"Error processing file {file.filename}: {str(e)}")
             raise
 

src/main.py

@@ -21,6 +21,7 @@ from src.utils.google_drive_service import GoogleDriveService
 # Import custom modules1
 #from src.agents.rag_agent import RAGAgent
 from src.agents.system_instructions_rag import SystemInstructionsRAGAgent
+from src.agents.rag_agent_manager import rag_agent_manager
 from src.models.document import AllDocumentsResponse, StoredDocument
 from src.models.UserContact import UserContactRequest
 from src.utils.document_processor import DocumentProcessor
@@ -48,7 +49,7 @@ app = FastAPI(title="Chatbot API")
 
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["http://localhost:8080"],  # Add your frontend URL
+    allow_origins=["http://localhost:8080", "http://localhost:3000"],  # Add both ports
     allow_credentials=True,
     allow_methods=["*"],  # Allows all methods
     allow_headers=["*"],  # Allows all headers
@@ -345,41 +346,14 @@ async def chat_endpoint(
         logger.info(f"Initializing LLM: {str(datetime.now())}")
         llm = get_llm_instance(request.llm_provider)
         
-        # Initialize RAG agent
-        # rag_agent = RAGAgent(
-        #     llm=llm,
-        #     embedding=embedding_model,
-        #     vector_store=vector_store,
-        #     mongodb=mongodb
-        # )
-
-        rag_agent = SystemInstructionsRAGAgent(
+        # Use RAG agent manager to get singleton RAG agent
+        rag_agent = rag_agent_manager.get_rag_agent(
             llm=llm,
-            embedding=embedding_model,
+            embedding_model=embedding_model,
             vector_store=vector_store,
             mongodb=mongodb
         )
         
-        # rag_agent.add_custom_role(
-        #     "Knowledge based chatbot and introduction specialist",
-        #     """You are a welcome agent with knowledge based specialist focusing on knowledge attached and create a beautiful welcome message.
-        #     Your role is to:
-        #     1. Your response should be short and to the point.
-        #     2. Strictly follow this point for If it is an introduction. You strictly respond that "Welcome name of customer to our platform. How can I help you today?"
-        #     """
-        # )
-
-        # rag_agent.add_custom_role(
-        #     "Knowledge based chatbot",
-        #     """You are a knowledge based specialist focusing on knowledge attached.
-        #     Your role is to:
-        #     1. Your response should be short and to the point.
-        #     2. if it is not introduction then make sure to share the response from Vector store.
-        #     3. If you do not find relevant information. Just say I do not have this information but this do not apply to introduction message.
-        #     4. If there is an introduction, you should ignore above roles and connect with LLm to have a welcome message for the user.
-        #     """
-        # )
-        
         # Use provided conversation ID or create new one
         conversation_id = request.conversation_id or str(uuid.uuid4())