1) Log google drive documents entry in mongo db

2) changed the embedding model to "BAAI/bge-large-en-v1.5"

config/config.py

@@ -27,15 +27,16 @@ class Settings:
     # Environment Configuration
     ENVIRONMENT = os.getenv('ENVIRONMENT').lower()
 
-    # Embedding Configuration
-    @property
-    def EMBEDDING_MODEL(self):
-        if self.ENVIRONMENT == 'production':
-            # Better model for demos
-            # return os.getenv('EMBEDDING_MODEL', 'openai/text-embedding-3-large')
-            return os.getenv('EMBEDDING_MODEL', 'all-MiniLM-L6-v2')
-        # Better for development purposes.
-        return os.getenv('EMBEDDING_MODEL', 'all-MiniLM-L6-v2')
+    # Embedding Configuration - Updated for BAAI model
+    EMBEDDING_MODEL = 'BAAI/bge-large-en-v1.5'
+    # @property
+    # def EMBEDDING_MODEL(self):
+    #     if self.ENVIRONMENT == 'production':
+    #         # Better model for demos
+    #         return os.getenv('EMBEDDING_MODEL', 'openai/text-embedding-3-large')
+    #         #return os.getenv('EMBEDDING_MODEL', 'all-MiniLM-L6-v2')
+    #     # Better for development purposes.
+    #     return os.getenv('EMBEDDING_MODEL', 'all-MiniLM-L6-v2')
 
     # MongoDB Configuration
     # MONGODB_URI = os.getenv('MONGODB_URI', 'mongodb://localhost:27017')

src/vectorstores/optimized_vectorstore.py

@@ -3,6 +3,10 @@ import asyncio
 from typing import Tuple, Optional, List, Dict, Any, Callable
 import concurrent.futures
 from functools import lru_cache
+import chromadb
+from chromadb.config import Settings
+import shutil
+import os
 
 from .base_vectorstore import BaseVectorStore
 from .chroma_vectorstore import ChromaVectorStore
@@ -10,11 +14,8 @@ from src.embeddings.huggingface_embedding import HuggingFaceEmbedding
 from src.utils.logger import logger
 from config.config import settings
 
+
 class OptimizedVectorStore(ChromaVectorStore):
-    """
-    Optimized vector store that maintains ChromaVectorStore compatibility
-    while adding caching and async initialization
-    """
     _instance: Optional['OptimizedVectorStore'] = None
     _lock = asyncio.Lock()
     _initialized = False
@@ -33,41 +34,11 @@ class OptimizedVectorStore(ChromaVectorStore):
         collection_name: str = "documents",
         client_settings: Optional[Dict[str, Any]] = None
     ):
-        """
-        Initialize the optimized vector store
-        Note: The actual initialization is deferred until needed
-        """
         if not self._initialized:
             self._persist_directory = persist_directory
             self._collection_name = collection_name
             self._client_settings = client_settings
             self._embedding_function = embedding_function
-            # Don't call super().__init__() here - we'll do it in _initialize()
-
-    @classmethod
-    async def create(
-        cls,
-        persist_directory: str = settings.CHROMA_PATH,
-        collection_name: str = "documents",
-        client_settings: Optional[Dict[str, Any]] = None
-    ) -> Tuple['OptimizedVectorStore', HuggingFaceEmbedding]:
-        """
-        Asynchronously create or get instance
-        
-        Returns:
-            Tuple[OptimizedVectorStore, HuggingFaceEmbedding]: 
-                The vector store instance and embedding model
-        """
-        async with cls._lock:
-            if not cls._instance or not cls._initialized:
-                instance = cls(
-                    persist_directory=persist_directory,
-                    collection_name=collection_name,
-                    client_settings=client_settings
-                )
-                await instance._initialize()
-                cls._instance = instance
-            return cls._instance, cls._instance._embedding_model
 
     async def _initialize(self) -> None:
         """Initialize the vector store and embedding model"""
@@ -77,21 +48,86 @@ class OptimizedVectorStore(ChromaVectorStore):
         try:
             # Load embedding model in background thread
             self._embedding_model = await self._load_embedding_model()
-            
-            # Initialize ChromaVectorStore with the loaded model
+
+            # Get embedding dimension
+            test_embedding = self._embedding_model.embed_query("test")
+            required_dim = len(test_embedding)
+
+            # Clean up existing database if dimensions don't match
+            await self._cleanup_if_needed(required_dim)
+
+            # Create ChromaDB client with fresh settings
+            client = chromadb.PersistentClient(
+                path=self._persist_directory,
+                settings=Settings(
+                    allow_reset=True,
+                    is_persistent=True,
+                    anonymized_telemetry=False
+                )
+            )
+
+            # Create new collection with correct dimensions
+            collection = client.create_collection(
+                name=self._collection_name,
+                metadata={
+                    "hnsw:space": "cosine",
+                    "hnsw:dim": required_dim
+                }
+            )
+
+            # Initialize parent class
             super().__init__(
                 embedding_function=self._embedding_model.embed_documents,
                 persist_directory=self._persist_directory,
-                collection_name=self._collection_name,
-                client_settings=self._client_settings
+                collection_name=self._collection_name
             )
-            
+
             self._initialized = True
-            
+            logger.info(
+                f"Successfully initialized vector store with dimension {required_dim}")
+
         except Exception as e:
             logger.error(f"Error initializing vector store: {str(e)}")
             raise
 
+    async def _cleanup_if_needed(self, required_dim: int) -> None:
+        """Clean up existing database if dimensions don't match"""
+        try:
+            # Create temporary client to check existing collection
+            temp_client = chromadb.PersistentClient(
+                path=self._persist_directory,
+                settings=Settings(allow_reset=True, is_persistent=True)
+            )
+
+            try:
+                # Try to get existing collection
+                collection = temp_client.get_collection(self._collection_name)
+                current_dim = collection.metadata.get(
+                    "hnsw:dim") if collection.metadata else None
+
+                if current_dim != required_dim:
+                    logger.info(
+                        f"Dimension mismatch: current={current_dim}, required={required_dim}")
+                    # Close client connection
+                    temp_client.reset()
+
+                    # Remove the entire directory
+                    if os.path.exists(self._persist_directory):
+                        shutil.rmtree(self._persist_directory)
+                        logger.info(
+                            f"Removed existing database at {self._persist_directory}")
+
+                    # Recreate empty directory
+                    os.makedirs(self._persist_directory, exist_ok=True)
+
+            except ValueError:
+                # Collection doesn't exist, no cleanup needed
+                pass
+
+        except Exception as e:
+            logger.error(f"Error during cleanup: {str(e)}")
+            raise
+
     async def _load_embedding_model(self) -> HuggingFaceEmbedding:
         """Load embedding model in background thread"""
         try:
@@ -110,28 +146,52 @@ class OptimizedVectorStore(ChromaVectorStore):
         """Create and cache embedding model"""
         return HuggingFaceEmbedding(model_name=settings.EMBEDDING_MODEL)
 
-    def __getattribute__(self, name):
-        """
-        Ensure initialization before accessing any ChromaVectorStore methods
-        """
-        # Get the attribute from the class
-        attr = super().__getattribute__(name)
-        
-        # If it's a method from ChromaVectorStore, ensure initialization
-        if callable(attr) and name in ChromaVectorStore.__dict__:
-            if not self._initialized:
-                raise RuntimeError(
-                    "Vector store not initialized. Please use 'await OptimizedVectorStore.create()'"
+    @classmethod
+    async def create(
+        cls,
+        persist_directory: str = settings.CHROMA_PATH,
+        collection_name: str = "documents",
+        client_settings: Optional[Dict[str, Any]] = None
+    ) -> Tuple['OptimizedVectorStore', HuggingFaceEmbedding]:
+        """Asynchronously create or get instance"""
+        async with cls._lock:
+            if not cls._instance or not cls._initialized:
+                instance = cls(
+                    persist_directory=persist_directory,
+                    collection_name=collection_name,
+                    client_settings=client_settings
                 )
-        return attr
+                await instance._initialize()
+                cls._instance = instance
+            return cls._instance, cls._instance._embedding_model
+
+    # Override parent class methods to ensure initialization
+    def add_documents(self, *args, **kwargs):
+        if not self._initialized:
+            raise RuntimeError("Vector store not initialized")
+        return super().add_documents(*args, **kwargs)
+
+    def similarity_search(self, *args, **kwargs):
+        if not self._initialized:
+            raise RuntimeError("Vector store not initialized")
+        return super().similarity_search(*args, **kwargs)
+
+    def get_document_chunks(self, *args, **kwargs):
+        if not self._initialized:
+            raise RuntimeError("Vector store not initialized")
+        return super().get_document_chunks(*args, **kwargs)
+
+    def delete_document(self, *args, **kwargs):
+        if not self._initialized:
+            raise RuntimeError("Vector store not initialized")
+        return super().delete_document(*args, **kwargs)
+
+    def get_all_documents(self, *args, **kwargs):
+        if not self._initialized:
+            raise RuntimeError("Vector store not initialized")
+        return super().get_all_documents(*args, **kwargs)
+
 
-# Factory function for getting optimized vector store
 async def get_optimized_vector_store() -> Tuple[ChromaVectorStore, HuggingFaceEmbedding]:
-    """
-    Get or create an optimized vector store instance
-    
-    Returns:
-        Tuple[ChromaVectorStore, HuggingFaceEmbedding]: 
-            The vector store and embedding model instances
-    """
-    return await OptimizedVectorStore.create()
+    """Get or create an optimized vector store instance"""
+    return await OptimizedVectorStore.create()