Updated Knowledge base with Upload, get and delete endpoints

src/db/mongodb_store.py

@@ -1,7 +1,6 @@
 # src/db/mongodb_store.py
 from motor.motor_asyncio import AsyncIOMotorClient
 from datetime import datetime
-import json
 from typing import List, Dict, Optional, Any
 from bson import ObjectId
 
@@ -9,25 +8,25 @@ class MongoDBStore:
     def __init__(self, mongo_uri: str = "mongodb://localhost:27017"):
         """Initialize MongoDB connection"""
         self.client = AsyncIOMotorClient(mongo_uri)
-        self.db = self.client.rag_chatbot
+        self.db = self.client.db_chatbot
         self.chat_history = self.db.chat_history
-        self.documents = self.db.documents  # Collection for original documents
+        self.documents = self.db.knowledge_base
 
     async def store_document(
         self,
         document_id: str,
         filename: str,
-        content: str,
         content_type: str,
-        file_size: int
+        file_size: int,
+        url_path: str
     ) -> str:
-        """Store original document in MongoDB"""
+        """Store document metadata in MongoDB"""
         document = {
             "document_id": document_id,
             "filename": filename,
-            "content": content,
             "content_type": content_type,
             "file_size": file_size,
+            "url_path": url_path,
             "upload_timestamp": datetime.now()
         }
         
@@ -38,12 +37,23 @@ class MongoDBStore:
         """Retrieve document by ID"""
         return await self.documents.find_one(
             {"document_id": document_id},
-            {"_id": 0}  # Exclude MongoDB's _id
+            {"_id": 0}
         )
 
     async def get_all_documents(self) -> List[Dict]:
         """Retrieve all documents"""
-        cursor = self.documents.find({}, {"_id": 0})
+        cursor = self.documents.find(
+            {},
+            {
+                "_id": 0,
+                "document_id": 1,
+                "filename": 1,
+                "content_type": 1,
+                "file_size": 1,
+                "url_path": 1,
+                "upload_timestamp": 1
+            }
+        )
         return await cursor.to_list(length=None)
 
     async def store_message(
@@ -117,4 +127,9 @@ class MongoDBStore:
                 'sources': doc['sources']
             })
             
-        return messages
+        return messages
+    
+    async def delete_document(self, document_id: str) -> bool:
+        """Delete document from MongoDB"""
+        result = await self.documents.delete_one({"document_id": document_id})
+        return result.deleted_count > 0

src/implementations/document_service.py

@@ -1,4 +1,5 @@
 # src/implementations/document_service.py
+from fastapi import HTTPException
 from pathlib import Path
 import shutil
 import os
@@ -20,8 +21,21 @@ class DocumentService:
     ):
         self.doc_processor = doc_processor
         self.mongodb = mongodb
-        self.upload_dir = Path("temp_uploads")
-        self.upload_dir.mkdir(exist_ok=True)
+        self.permanent_dir = Path("uploads")
+        self.permanent_dir.mkdir(exist_ok=True)
+
+    async def check_duplicate_filename(self, filename: str) -> bool:
+        """
+        Check if a file with the same name exists
+        
+        Args:
+            filename (str): Original filename to check
+            
+        Returns:
+            bool: True if duplicate exists, False otherwise
+        """
+        documents = await self.mongodb.get_all_documents()
+        return any(doc.get('filename') == filename for doc in documents)
 
     async def process_documents(
         self,
@@ -54,6 +68,14 @@ class DocumentService:
 
         for file in files:
             try:
+                # Check for duplicate filename
+                if await self.check_duplicate_filename(file.filename):
+                    failed_files.append(self._create_failed_file_entry(
+                        file.filename,
+                        "A document with this name already exists. Please upload another document."
+                    ))
+                    continue
+
                 if not self._is_supported_format(file.filename):
                     failed_files.append(self._create_failed_file_entry(
                         file.filename, 
@@ -86,30 +108,31 @@ class DocumentService:
         """Process a single file upload"""
         # Generate UUID for document
         document_id = str(uuid4())
-        temp_path = self.upload_dir / f"{document_id}_{file.filename}"
+        filename = f"{document_id}_{file.filename}"
+        file_path = self.permanent_dir / filename
+        url_path = f"/docs/{filename}"
         
         try:
-            # Save file temporarily
-            with open(temp_path, "wb") as buffer:
+            # Save file to permanent location
+            with open(file_path, "wb") as buffer:
                 shutil.copyfileobj(file.file, buffer)
 
-            # Process the document to get content and metadata
-            processed_doc = await self.doc_processor.process_document(temp_path)
-            content = processed_doc['content']
+            # Process the document for vector store
+            processed_doc = await self.doc_processor.process_document(file_path)
             
-            # First, store in MongoDB
+            # Store in MongoDB with url_path
             await self.mongodb.store_document(
                 document_id=document_id,
                 filename=file.filename,
-                content=content,
                 content_type=file.content_type,
-                file_size=os.path.getsize(temp_path)
+                file_size=os.path.getsize(file_path),
+                url_path=url_path
             )
 
-            # Then process for vector store in background
+            # Process for vector store in background
             background_tasks.add_task(
                 self._process_for_vector_store,
-                processed_doc['chunks'],  # Use the chunks from processed document
+                processed_doc['chunks'],
                 vector_store,
                 document_id,
                 file.filename
@@ -121,18 +144,32 @@ class DocumentService:
                 status="processing",
                 document_info=DocumentInfo(
                     original_filename=file.filename,
-                    size=os.path.getsize(temp_path),
-                    content_type=file.content_type
+                    size=os.path.getsize(file_path),
+                    content_type=file.content_type,
+                    url_path=url_path
                 )
             )
-        finally:
-            # Clean up temporary file
-            if temp_path.exists():
-                temp_path.unlink()
+            
+        except Exception as e:
+            # Clean up file if it was created
+            if file_path.exists():
+                try:
+                    file_path.unlink()
+                except Exception as cleanup_error:
+                    logger.error(f"Error cleaning up file {file_path}: {str(cleanup_error)}")
+            
+            # Clean up from MongoDB if document was created
+            try:
+                await self.mongodb.delete_document(document_id)
+            except Exception as db_cleanup_error:
+                logger.error(f"Error cleaning up MongoDB document {document_id}: {str(db_cleanup_error)}")
+            
+            logger.error(f"Error processing file {file.filename}: {str(e)}")
+            raise
 
     async def _process_for_vector_store(
         self,
-        chunks: List[str],  # Now accepting pre-processed chunks
+        chunks: List[str],
         vector_store: ChromaVectorStore,
         document_id: str,
         filename: str
@@ -147,7 +184,7 @@ class DocumentService:
             
             # Prepare metadata for each chunk
             metadatas = [{
-                'document_id': document_id,  # MongoDB document ID
+                'document_id': document_id,
                 'source_file': filename,
                 'chunk_index': i,
                 'total_chunks': len(chunks)
@@ -179,7 +216,29 @@ class DocumentService:
             "error": error
         }
 
+    async def delete_document(self, document_id: str) -> bool:
+        """Delete document from storage and MongoDB"""
+        try:
+            # Get document details from MongoDB
+            doc = await self.mongodb.get_document(document_id)
+            if doc:
+                # Get filename from url_path
+                filename = doc['url_path'].split('/')[-1]
+                file_path = self.permanent_dir / filename
+                
+                # Delete physical file if it exists
+                if file_path.exists():
+                    file_path.unlink()
+                
+                # Delete from MongoDB
+                return await self.mongodb.delete_document(document_id)
+            return False
+            
+        except Exception as e:
+            logger.error(f"Error deleting document: {str(e)}")
+            raise
+
     def cleanup(self):
-        """Clean up upload directory"""
-        if self.upload_dir.exists() and not any(self.upload_dir.iterdir()):
-            self.upload_dir.rmdir()
+        """Clean up permanent directory if empty"""
+        if self.permanent_dir.exists() and not any(self.permanent_dir.iterdir()):
+            self.permanent_dir.rmdir()

src/main.py

@@ -1,9 +1,12 @@
 # src/main.py
 from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks
-from fastapi.responses import StreamingResponse
+from fastapi.responses import StreamingResponse, FileResponse
+from fastapi.staticfiles import StaticFiles
 from typing import List
 import uuid
 from datetime import datetime
+from pathlib import Path
+import os
 
 # Import custom modules
 from src.agents.rag_agent import RAGAgent
@@ -25,20 +28,82 @@ from src.models import (
 )
 from config.config import settings
 
-app = FastAPI(title="RAG Chatbot API")
+app = FastAPI(title="Chatbot API")
 
 # Initialize MongoDB
 mongodb = MongoDBStore(settings.MONGODB_URI)
 
 # Initialize core components
-doc_processor = DocumentProcessor(
-    chunk_size=1000,
-    chunk_overlap=200,
-    max_file_size=10 * 1024 * 1024
-)
+doc_processor = DocumentProcessor()
 summarizer = ConversationSummarizer()
 document_service = DocumentService(doc_processor, mongodb)
 
+# Create uploads directory if it doesn't exist
+UPLOADS_DIR = Path("uploads")
+UPLOADS_DIR.mkdir(exist_ok=True)
+
+# Mount the uploads directory for static file serving
+app.mount("/docs", StaticFiles(directory=str(UPLOADS_DIR)), name="documents")
+
+@app.get("/documents")
+async def get_all_documents():
+    """Get all documents from MongoDB"""
+    try:
+        documents = await mongodb.get_all_documents()
+        
+        formatted_documents = []
+        for doc in documents:
+            try:
+                formatted_doc = {
+                    "document_id": doc.get("document_id"),
+                    "filename": doc.get("filename"),
+                    "content_type": doc.get("content_type"),
+                    "file_size": doc.get("file_size"),
+                    "url_path": doc.get("url_path"),
+                    "upload_timestamp": doc.get("upload_timestamp")
+                }
+                formatted_documents.append(formatted_doc)
+            except Exception as e:
+                logger.error(f"Error formatting document {doc.get('document_id', 'unknown')}: {str(e)}")
+                continue
+        
+        return {
+            "total_documents": len(formatted_documents),
+            "documents": formatted_documents
+        }
+    except Exception as e:
+        logger.error(f"Error retrieving documents: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/documents/{document_id}/download")
+async def get_document_file(document_id: str):
+    """Serve a document file by its ID"""
+    try:
+        # Get document info from MongoDB
+        doc = await mongodb.get_document(document_id)
+        if not doc:
+            raise HTTPException(status_code=404, detail="Document not found")
+        
+        # Extract filename from url_path
+        filename = doc["url_path"].split("/")[-1]
+        file_path = UPLOADS_DIR / filename
+        
+        if not file_path.exists():
+            raise HTTPException(
+                status_code=404, 
+                detail=f"File not found on server: {filename}"
+            )
+            
+        return FileResponse(
+            path=str(file_path),
+            filename=doc["filename"],
+            media_type=doc["content_type"]
+        )
+            
+    except Exception as e:
+        logger.error(f"Error serving document file: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
 @app.post("/documents/upload", response_model=BatchUploadResponse)
 async def upload_documents(
     files: List[UploadFile] = File(...),
@@ -55,36 +120,7 @@ async def upload_documents(
         return response
     except Exception as e:
         logger.error(f"Error in document upload: {str(e)}")
-        raise HTTPException(status_code=500, detail=str(e))
-    finally:
-        document_service.cleanup()
-
-@app.get("/documents", response_model=AllDocumentsResponse)
-async def get_all_documents(include_embeddings: bool = False):
-    """
-    Get all documents stored in the system
-    
-    Args:
-        include_embeddings (bool): Whether to include embeddings in the response
-    """
-    try:
-        vector_store, _ = await get_vector_store()
-        documents = vector_store.get_all_documents(include_embeddings=include_embeddings)
-        
-        return AllDocumentsResponse(
-            total_documents=len(documents),
-            documents=[
-                StoredDocument(
-                    id=doc['id'],
-                    text=doc['text'],
-                    embedding=doc.get('embedding'),
-                    metadata=doc.get('metadata')
-                ) for doc in documents
-            ]
-        )
-    except Exception as e:
-        logger.error(f"Error retrieving documents: {str(e)}")
-        raise HTTPException(status_code=500, detail=str(e))
+        raise HTTPException(status_code=500, detail=str(e))    
 
 @app.get("/documentchunks/{document_id}")
 async def get_document_chunks(document_id: str):
@@ -104,6 +140,47 @@ async def get_document_chunks(document_id: str):
     except Exception as e:
         logger.error(f"Error retrieving document chunks: {str(e)}")
         raise HTTPException(status_code=500, detail=str(e))
+
+@app.delete("/documents/{document_id}")
+async def delete_document(document_id: str):
+    """Delete document from MongoDB, ChromaDB, and physical storage"""
+    try:
+        # First get document details from MongoDB to get file path
+        document = await mongodb.get_document(document_id)
+        if not document:
+            raise HTTPException(status_code=404, detail="Document not found")
+        
+        # Get vector store instance
+        vector_store, _ = await get_vector_store()
+        
+        # Delete physical file using document service
+        deletion_success = await document_service.delete_document(document_id)
+        if not deletion_success:
+            logger.warning(f"Failed to delete physical file for document {document_id}")
+            
+        # Delete from vector store
+        try:
+            vector_store.delete_document(document_id)
+        except Exception as e:
+            logger.error(f"Error deleting document from vector store: {str(e)}")
+            raise HTTPException(
+                status_code=500, 
+                detail=f"Failed to delete document from vector store: {str(e)}"
+            )
+            
+        # Delete from MongoDB - don't check return value since document might already be deleted
+        await mongodb.delete_document(document_id)
+        
+        return {
+            "status": "success",
+            "message": f"Document {document_id} successfully deleted from all stores"
+        }
+        
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error in delete_document endpoint: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
     
 @app.post("/chat", response_model=ChatResponse)
 async def chat_endpoint(

src/models/document.py

@@ -1,4 +1,4 @@
-# src/models/document.py
+#  src/models/document.py
 from pydantic import BaseModel
 from typing import Optional, List, Dict, Any
 
@@ -7,6 +7,7 @@ class DocumentInfo(BaseModel):
     original_filename: str
     size: int
     content_type: str
+    url_path: str
 
 class DocumentResponse(BaseModel):
     """Response model for document processing"""

src/utils/logger.py

@@ -4,7 +4,7 @@ import sys
 from typing import Optional
 
 def setup_logger(
-    name: str = "rag_chatbot", 
+    name: str = "chatbot", 
     log_level: str = "INFO",
     log_file: Optional[str] = None
 ) -> logging.Logger: