Create bot service

alembic/versions/f00987c5ad39_describe_bot_name.py

@@ -0,0 +1,30 @@
+"""describe bot name
+
+Revision ID: f00987c5ad39
+Revises: b1df0377cbe2
+Create Date: 2024-10-15 12:56:39.589516
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'f00987c5ad39'
+down_revision: Union[str, None] = 'b1df0377cbe2'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('session_publisher', sa.Column('bot_name', sa.String(length=100), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('session_publisher', 'bot_name')
+    # ### end Alembic commands ###

api/auth.py

@@ -1,17 +1,19 @@
+import os
+from datetime import timedelta, datetime, timezone
+from typing import Annotated
+
+from dotenv import load_dotenv
 from fastapi import APIRouter, Depends
 from fastapi.responses import JSONResponse
 from fastapi.security import OAuth2PasswordBearer
-from dotenv import load_dotenv
-
+from jose import jwt, JWTError
+from passlib.context import CryptContext
 from sqlalchemy.orm import Session
-from db.models import User
 from starlette import status
-from datetime import timedelta, datetime, timezone
+
+from db.models import User
 from db.database import get_db
-from passlib.context import CryptContext
-from typing import Annotated
-from jose import jwt, JWTError
-import os
+
 
 load_dotenv()
 
@@ -68,4 +70,17 @@ async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]):
     except JWTError:
         return JSONResponse(
             status_code=status.HTTP_401_UNAUTHORIZED, content="Could not validate user."
-        )
+        )
+
+user_dependency = Annotated[dict, Depends(get_current_user)]
+def check_user_authentication(user: user_dependency):
+    """Helper function to check if the user is authenticated."""
+    if user is None:
+        return JSONResponse(status_code=401, content="Authentication Failed")
+    return None
+
+def check_admin_authentication(user: user_dependency):
+    """Helper function to check if the user is authenticated."""
+    if user is None or user.get("role_id") != 1:
+        return JSONResponse(status_code=401, content="Authentication Admin Failed")
+    return None

api/events.py

@@ -1,10 +1,12 @@
+import os
+
+from dotenv import load_dotenv
 from fastapi import FastAPI
-from db.models import Base
+
 from db.database import engine
-from api.router.book import db_conn
+from db.models import Base
 from llama_index.core import set_global_handler
-from dotenv import load_dotenv
-import os
+
 
 
 load_dotenv()
@@ -12,8 +14,6 @@ load_dotenv()
 
 async def startup() -> None:
     Base.metadata.create_all(engine)
-    print("table added")
-    await db_conn.connect()
     os.environ["LANGFUSE_SECRET_KEY"] = os.getenv("LANGFUSE_SECRET_KEY")
     os.environ["LANGFUSE_PUBLIC_KEY"] = os.getenv("LANGFUSE_PUBLIC_KEY")
     os.environ["LANGFUSE_HOST"] = os.getenv("LANGFUSE_HOST")
@@ -21,7 +21,7 @@ async def startup() -> None:
 
 
 async def shutdown() -> None:
-    await db_conn.disconnect()
+    pass
 
 def register_events(app: FastAPI) -> FastAPI:
     app.add_event_handler("startup", startup)

api/function.py

@@ -1,31 +1,19 @@
-from script.vector_db import IndexManager
-from script.document_uploader import Uploader
-from db.get_data import GetDatabase
-from db.delete_data import DeleteDatabase
-from db.update_data import UpdateDatabase
-
+import json
+import logging
 from typing import Any
-from fastapi import UploadFile
-from fastapi import HTTPException
+
+from dotenv import load_dotenv
+from fastapi import HTTPException, UploadFile
 from fastapi.responses import JSONResponse
 
-from llama_index.core.llms import MessageRole
 from core.chat.engine import Engine
-from core.chat.chatstore import ChatStore
-from core.parser import clean_text, update_response, renumber_sources
-from service.dto import BotResponseStreaming, ChatMessage
+from core.parser import clean_text
+from langfuse.llama_index import LlamaIndexCallbackHandler
+from script.document_uploader import Uploader
+from script.vector_db import IndexManager
 from service.aws_loader import Loader
+from service.dto import BotResponseStreaming
 
-from pymongo.mongo_client import MongoClient
-from dotenv import load_dotenv
-
-from typing import List
-from datetime import datetime
-import redis
-import logging
-import re
-import json
-import os
 
 load_dotenv()
 
@@ -33,8 +21,14 @@ load_dotenv()
 logging.basicConfig(level=logging.INFO)
 
 
-async def data_ingestion(category_id, reference, file: UploadFile) -> Any:
+async def data_ingestion(reference, file: UploadFile) -> Any:
     try:
+        # Assuming you have a Langfuse callback handler
+        langfuse_callback_handler = LlamaIndexCallbackHandler()
+        langfuse_callback_handler.set_trace_params(
+            user_id="admin_book_uploaded",
+        )
+        
         # Upload to AWS
         file_name = f"{reference['title']}"
         aws_loader = Loader()
@@ -62,186 +56,6 @@ async def data_ingestion(category_id, reference, file: UploadFile) -> Any:
             content="An internal server error occurred in data ingestion.",
         )
 
-
-async def get_data(db_conn, title=None, fetch_all_data=True):
-    get_database = GetDatabase(db_conn)
-    print(get_database)
-    try:
-        if fetch_all_data:
-            results = await get_database.get_all_data()
-            print(results)
-            logging.info("Database fetched all data")
-            return results
-        else:
-            results = await get_database.get_data(title)
-            logging.info("Database fetched one data")
-            return results
-
-    except Exception as e:
-        # Log the error and raise HTTPException for FastAPI
-        logging.error("An error occurred in get data: %s", e)
-        return JSONResponse(
-            status_code=500, content="An internal server error occurred in get data."
-        )
-
-
-async def update_data(id: int, reference, db_conn):
-    update_database = UpdateDatabase(db_conn)
-    try:
-        reference = reference.model_dump()
-        print(reference)
-        reference.update({"id": id})
-        print(reference)
-        await update_database.update_record(reference)
-        response = {"status": "Update Success"}
-        return response
-    except Exception as e:
-        # Log the error and raise HTTPException for FastAPI
-        logging.error("An error occurred in update data: %s", e)
-        return JSONResponse(
-            status_code=500, content="An internal server error occurred in update data."
-        )
-
-
-async def delete_data(id: int, db_conn):
-    delete_database = DeleteDatabase(db_conn)
-    try:
-        params = {"id": id}
-        await delete_database.delete_record(params)
-        response = {"status": "Delete Success"}
-        return response
-    except Exception as e:
-        # Log the error and raise HTTPException for FastAPI
-        logging.error("An error occurred in get data: %s", e)
-        return JSONResponse(
-            status_code=500, content="An internal server error occurred in delete data."
-        )
-
-
-def generate_completion_non_streaming(
-    session_id, user_request, titles: List = None, type_bot="general"
-):
-    uri = os.getenv("MONGO_URI")
-    engine = Engine()
-    index_manager = IndexManager()
-    chatstore = ChatStore()
-    client = MongoClient(uri)
-
-    try:
-        client.admin.command("ping")
-        print("Pinged your deployment. You successfully connected to MongoDB!")
-    except Exception as e:
-        return JSONResponse(status_code=500, content=f"Database Error as {e}")
-
-    try:
-
-        # Load existing indexes
-        index = index_manager.load_existing_indexes()
-
-        if type_bot == "general":
-            # Retrieve the chat engine with the loaded index
-            chat_engine = engine.get_chat_engine(session_id, index)
-        else:
-            # Retrieve the chat engine with the loaded index
-            chat_engine = engine.get_chat_engine(session_id, index, titles, type_bot)
-
-        # Generate completion response
-        response = chat_engine.chat(user_request)
-
-        sources = response.sources
-
-        number_reference = list(set(re.findall(r"\[(\d+)\]", str(response))))
-        number_reference_sorted = sorted(number_reference)
-
-        contents = []
-        metadata_collection = []
-        scores = []
-
-        if number_reference_sorted:
-            for number in number_reference_sorted:
-                # Konversi number ke integer untuk digunakan sebagai indeks
-                number = int(number)
-
-                # Pastikan sources tidak kosong dan memiliki elemen yang diperlukan
-                if sources and len(sources) > 0:
-                    node = dict(sources[0])["raw_output"].source_nodes
-
-                    # Pastikan number valid sebagai indeks
-                    if 0 <= number - 1 < len(node):
-
-                        content = clean_text(node[number - 1].node.get_text())
-                        contents.append(content)
-
-                        metadata = dict(node[number - 1].node.metadata)
-                        metadata_collection.append(metadata)
-
-                        score = node[number - 1].score
-                        scores.append(score)
-                    else:
-                        print(f"Invalid reference number: {number}")
-                else:
-                    print("No sources available")
-        else:
-            print("There are no references")
-
-        response = update_response(str(response))
-        contents = renumber_sources(contents)
-
-        # Check the lengths of content and metadata
-        num_content = len(contents)
-        num_metadata = len(metadata_collection)
-
-        # Add content to metadata
-        for i in range(min(num_content, num_metadata)):
-            metadata_collection[i]["content"] = re.sub(r"source \d+\:", "", contents[i])
-
-        message = ChatMessage(
-            role=MessageRole.ASSISTANT, content=response, metadata=metadata_collection
-        )
-
-        chatstore.delete_last_message(session_id)
-        chatstore.add_message(session_id, message)
-        chatstore.clean_message(session_id)
-    except Exception as e:
-        # Log the error and raise HTTPException for FastAPI
-        logging.error("An error occurred in generate text: %s", e)
-        return JSONResponse(
-            status_code=500,
-            content=f"An internal server error occurred in generate text as {e}.")
-
-    try :
-        chat_history_db = [
-                ChatMessage(role=MessageRole.SYSTEM, 
-                            content=user_request, 
-                            timestamp=datetime.now(),
-                            payment = "free" if type_bot=="general" else None
-                            ),
-                ChatMessage(
-                    role=MessageRole.ASSISTANT,
-                    content=response,
-                    metadata=metadata_collection,
-                    timestamp=datetime.now(),
-                    payment = "free" if type_bot=="general" else None 
-                )
-            ]
-        
-        chat_history_json = [message.model_dump() for message in chat_history_db]
-
-        db = client["bot_database"]  # Replace with your database name
-        collection = db[session_id]  # Replace with your collection name
-        
-        result = collection.insert_many(chat_history_json)
-        print("Data inserted with record ids", result.inserted_ids)
-
-        return str(response), metadata_collection, scores
-    
-    except Exception as e:
-        # Log the error and raise HTTPException for FastAPI
-        logging.error("An error occurred in generate text: %s", e)
-        return JSONResponse(
-            status_code=500,
-            content=f"An internal server error occurred in generate text as {e}.")
-
 async def generate_streaming_completion(user_request, session_id):
     try:
         engine = Engine()

api/router/book.py

@@ -1,32 +1,53 @@
 import logging
-import base64
-
 from typing import Annotated, Optional
-from api.function import data_ingestion, get_data, delete_data, update_data
+
+from api.function import data_ingestion
 from api.router.user import user_dependency
+from api.auth import check_user_authentication, check_admin_authentication
+
 from fastapi import Form, APIRouter, File, UploadFile, Depends
 from fastapi.responses import JSONResponse
-from db.repository import get_db_conn
-from db.get_data import GetDatabase
-from db.models import Category, Metadata
+
+from db.models import Metadata
 from db.database import get_db
-from langfuse.llama_index import LlamaIndexCallbackHandler
+from db.query.query_book import BookQuery
+from db.query.query_category import CategoryQuery
+from db.fetching import DataFetching
+
 from config import MYSQL_CONFIG
+from utils.error_handlers import handle_exception
 from script.vector_db import IndexManager
-from service.dto import MetadataRequest, MetadataResponse
+from service.dto import MetadataResponse
 from sqlalchemy.orm import Session
 from sqlalchemy.future import select
-from sqlalchemy.exc import SQLAlchemyError
 
 
 router = APIRouter(tags=["Book"])
 
-db_conn = get_db_conn(MYSQL_CONFIG)
-get_database = GetDatabase(db_conn)
 index_manager = IndexManager()
 db_dependency = Annotated[Session, Depends(get_db)]
 
 
+@router.get("/book")
+async def get_metadata(user: user_dependency, db: db_dependency):
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        print(auth_response)
+        return auth_response
+    try:
+        # Join Metadata with Category to get the category name
+        fetching = DataFetching(user, db)
+        # print(fetching)
+        metadata_fetching = fetching.metadata_fetching()
+        # print(metadata_fetching)
+
+        # Transform results into MetadataResponse model with optional thumbnail handling
+        return metadata_fetching
+
+    except Exception as e:
+        return handle_exception(e)
+
+
 @router.post("/book")
 async def upload_file(
     user: user_dependency,
@@ -39,54 +60,21 @@ async def upload_file(
     file: UploadFile = File(...),
     thumbnail: Optional[UploadFile] = File(None),
 ):
-    print(user.get("role_id"))
-    # if user is None or user.get('role_id') != 1:
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_admin_authentication(user)
+    if auth_response:
+        return auth_response
 
     # Create a new Metadata object
-    new_book = Metadata(
-        title=title,
-        author=author,
-        category_id=category_id,
-        year=year,
-        publisher=publisher,
-    )
-
-    db.add(new_book)
-    db.commit()
-    logging.info("Database Inserted")
-
-    try:
-        # Query the category based on category_id
-        category_query = select(Category.category).where(Category.id == category_id)
-        result = db.execute(category_query)
-        category = result.scalar_one_or_none()
+    book_query = BookQuery(user)
+    book_query.add_book(db, title, author, category_id, year, publisher)
 
-        # Check if the category exists
-        if category is None:
-            return JSONResponse(status_code=404, content="Category not found")
-
-    except SQLAlchemyError as db_exc:
-        # Handle any database-related errors (e.g., connection issues, query issues)
-        print(f"Database error: {db_exc}")
-        return JSONResponse(status_code=500, content="Database error occurred")
+    logging.info("Database Inserted")
 
-    except Exception as e:
-        # Catch any other general exceptions
-        print(f"Error: {e}")
-        return JSONResponse(
-            status_code=500, content="An error occurred while processing your request"
-        )
+    # Query the category based on category_id
+    category_query = CategoryQuery(user)
+    category = category_query.get_category(db, category_id)
 
     try:
-        # Assuming you have a Langfuse callback handler
-        langfuse_callback_handler = LlamaIndexCallbackHandler()
-        langfuse_callback_handler.set_trace_params(
-            user_id="admin_book_uploaded",
-        )
-
-        # Construct the reference dictionary
         reference = {
             "title": title,
             "author": author,
@@ -96,61 +84,16 @@ async def upload_file(
         }
 
         # Process the file and handle data ingestion
-        response = await data_ingestion(category_id, reference, file)
-
-    except Exception as e:
-        # Handle any errors related to file processing or data ingestion
-        print(f"File processing error: {e}")
-        return JSONResponse(status_code=500, content="File processing error")
-
-    # Return a successful response with the uploaded filename and response from data ingestion
-    return {
-        "filename": file.filename,
-        "response": response,
-        "info": "upload file successfully",
-    }
-
+        response = await data_ingestion(reference, file)
 
-@router.get("/book")
-async def get_metadata(user: user_dependency, db: db_dependency):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
-    try:
-        # Join Metadata with Category to get the category name
-        results = (
-            db.query(
-                Metadata.id,
-                Metadata.title,
-                Metadata.author,
-                Category.category,  # Assuming this is the correct field for category name
-                Category.id,
-                Metadata.year,
-                Metadata.publisher,
-                Metadata.thumbnail,
-            )
-            .join(Category, Metadata.category_id == Category.id)
-            .all()
-        )
+        return {
+            "filename": file.filename,
+            "response": response,
+            "info": "upload file successfully",
+        }
 
-        # Transform results into MetadataResponse model with optional thumbnail handling
-        return [
-            MetadataResponse(
-                id = id,
-                title=title,
-                author=author,
-                category=category,
-                category_id = category_id,
-                year=year,
-                publisher=publisher,
-                thumbnail=(
-                    thumbnail if thumbnail else None
-                ),  # Ensure None if thumbnail is not present
-            )
-            for id, title, author, category, category_id, year, publisher, thumbnail in results
-        ]
-    except SQLAlchemyError as db_exc:
-        print(f"Database error: {db_exc}")
-        return JSONResponse(status_code=500, content="Database error occurred")
+    except Exception as e:
+        return handle_exception(e)
 
 
 @router.put("/book/{metadata_id}")
@@ -165,38 +108,29 @@ async def update_metadata(
     publisher: str = Form(...),
     thumbnail: Optional[UploadFile] = File(None),
 ):
-    if user is None or user.get("role_id") != 1:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_admin_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
-        # old_metadata = await get_database.get_data_by_id(metadata_id)
-
-        old_metadata = db.execute(
-            select(Metadata).where(Metadata.id == metadata_id)
-        ).scalar_one_or_none()
-
-        if old_metadata is None:
-            return JSONResponse(status_code=404, content="Metadata not found.")
-
-        # Fetch old and new categories
-        old_category = db.execute(
-            select(Category.category).join(Metadata).where(Metadata.id == metadata_id)
-        ).scalar_one_or_none()
+        # fetch current metadata
+        book_query = BookQuery(user)
+        current_metadata = book_query.get_metadata_books(db, metadata_id)
 
-        new_category = db.execute(
-            select(Category.category).where(Category.id == category_id)
-        ).scalar_one_or_none()
+        # Fetch current and new categories
+        category_query = CategoryQuery(user)
+        current_category = category_query.get_current_category(db, metadata_id)
+        
+        new_category = category_query.get_category(db, category_id)
 
-        if old_category is None or new_category is None:
-            return JSONResponse(status_code=404, content="Category not found.")
 
         # Prepare the references
-        old_reference = {
-            "title": old_metadata.title,
-            "author": old_metadata.author,
-            "category": old_category,
-            "year": old_metadata.year,
-            "publisher": old_metadata.publisher,
+        current_reference = {
+            "title": current_metadata.title,
+            "author": current_metadata.author,
+            "category": current_category,
+            "year": current_metadata.year,
+            "publisher": current_metadata.publisher,
         }
 
         new_reference = {
@@ -207,72 +141,62 @@ async def update_metadata(
             "publisher": publisher,
         }
 
-        index_manager.update_vector_database(old_reference, new_reference)
+        # Update vector database
+        index_manager.update_vector_database(current_reference, new_reference)
 
         # Update existing metadata entry
         metadata = db.query(Metadata).filter(Metadata.id == metadata_id).first()
+        print(metadata)
 
         if not metadata:
             return JSONResponse(status_code=404, content="Metadata not found")
 
-        if thumbnail is not None:
-            thumbnail_data = await thumbnail.read()  # Read the file data
-            metadata.thumbnail = base64.b64encode(thumbnail_data).decode("utf-8")
-
-        # Update fields only if provided in the request
-        metadata.title = title
-        metadata.author = author
-        metadata.category_id = category_id
-        metadata.year = year
-        metadata.publisher = publisher
-
-        # Commit the changes to the database
-        db.commit()
-        db.refresh(metadata)  # Refresh to get the updated data
-
-        category_query = select(Category.category).where(
-            Category.id == metadata.category_id
+        updated_metadata = book_query.update_metadata_entry(
+            db, metadata_id, title, author, category_id, year, publisher
         )
-        result = db.execute(category_query)
-        category = result.scalar_one_or_none()
+        print(updated_metadata)
+        updated_category = category_query.get_category(db, updated_metadata.category_id)
+        print(updated_category)
 
         return MetadataResponse(
             id=metadata_id,
-            title=metadata.title,
-            author=metadata.author,
-            category=category,  # Assuming category relationship is available
-            category_id=metadata.category_id,
-            year=metadata.year,
-            publisher=metadata.publisher,
-            thumbnail=metadata.thumbnail if metadata.thumbnail is not None else None,
+            title=updated_metadata.title,
+            author=updated_metadata.author,
+            category=updated_category,  # Assuming category relationship is available
+            category_id=updated_metadata.category_id,
+            year=updated_metadata.year,
+            publisher=updated_metadata.publisher,
+            thumbnail=(
+                updated_metadata.thumbnail
+                if updated_metadata.thumbnail is not None
+                else None
+            ),
         )
 
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An error occurred while updating metadata as {e}"
-        )
-
+        return handle_exception(e)
 
 @router.delete("/book/{metadata_id}")
 async def delete_metadata(user: user_dependency, db: db_dependency, metadata_id: int):
-    if user is None or user.get("role_id") != 1:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_admin_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
-        old_reference = await get_database.get_data_by_id(metadata_id)
-        
-        old_metadata = db.execute(
-            select(Metadata).where(Metadata.id == metadata_id)
-        ).scalar_one_or_none()
-        
-        index_manager.delete_vector_database(old_reference)
+        # Check if metadata exists before deletion
+        metadata = db.query(Metadata).filter(Metadata.id == metadata_id).first()
 
-        await delete_data(metadata_id, db_conn)
+        if not metadata:
+            return JSONResponse(status_code=404, content="Metadata not found")
+
+        # Delete from the vector database and the database
+        index_manager.delete_vector_database(metadata.title)
+        
+        db.delete(metadata)
+        db.commit()
 
         return {"Status": "delete successfully"}
 
     except Exception as e:
-        print(e)
-        return JSONResponse(
-            status_code=500, content="An error occurred while delete metadata"
-        )
+        return handle_exception(e)
+    

api/router/book_collection.py

@@ -1,188 +1,105 @@
 from typing import Annotated, List, Optional
 from api.router.user import user_dependency
+from api.auth import check_user_authentication
 from fastapi import APIRouter, Depends
-from fastapi.responses import JSONResponse
-from db.models import User_Meta, Metadata, Category
 from db.database import get_db
 from sqlalchemy.orm import Session
-from sqlalchemy.exc import SQLAlchemyError
+from db.query.query_user_meta import UserMetaQuery
+from db.fetching import DataFetching
+from utils.error_handlers import handle_exception
 
-router = APIRouter(tags=["Book_Collection"])
 
+router = APIRouter(tags=["Book_Collection"])
 db_dependency = Annotated[Session, Depends(get_db)]
 
 
 @router.get("/book_collection")
-async def get_book_collection(
-    user: user_dependency,
-    db: db_dependency,
-):
-    """This function will return a BookCollection"""
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+async def get_book_collection(user: user_dependency, db: db_dependency):
+    """Fetch user's book collection."""
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
-        # Fetch all User_Meta entries for the user and their associated Metadata
-        user_meta_entries = (
-            db.query(User_Meta, Metadata, Category)  # Select User_Meta, Metadata, and Category
-            .join(Metadata, Metadata.id == User_Meta.metadata_id)  # Join on metadata_id
-            .join(Category, Category.id == Metadata.category_id)  # Join on category_id
-            .filter(User_Meta.user_id == user.get("id"))
-            .all()
-        )
-
-        if not user_meta_entries:
-            return {"info": "No book collection found"}
-
-        # Extract relevant data from the user_meta_entries
-        results = [
-            {
-                "user_id": user_meta.user_id,
-                "metadata_id": metadata.id,  # Access Metadata fields
-                "title": metadata.title,  # Replace with actual field names in Metadata
-                "author": metadata.author,  # Replace with actual field names in Metadata
-                "category_name": category.category,  # Replace with actual field names in Category
-                "year": metadata.year,
-                "publisher": metadata.publisher
-                # Add other Metadata and Category fields as needed
-            }
-            for user_meta, metadata, category in user_meta_entries  # Unpack the tuple
-        ]
 
+        fetching = DataFetching(user, db)
+        book_collection = fetching.collection_fetching()
+        
         return {
             "status": "success",
-            "book_collection": results,  # Return the list of metadata
+            "book_collection": book_collection,
         }
-
-    except SQLAlchemyError as e:
-        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
+        return handle_exception(e)
 
 
 @router.post("/book_collection")
 async def request_book_collection(
-    user: user_dependency,
-    db: db_dependency,
-    metadata_id: List[Optional[int]],
+    user: user_dependency, db: db_dependency, metadata_ids: List[Optional[int]]
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    """Insert book collection metadata for a user."""
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
-    try:
-        # Create User_Meta instances for each metadata_id
-        user_meta_entries = [
-            User_Meta(user_id=user.get("id"), metadata_id=mid) for mid in metadata_id
-        ]
+    user_meta_query = UserMetaQuery(user)
 
-        # Insert all entries into the database
-        db.add_all(user_meta_entries)
-        db.commit()  # Commit the transaction
+    try:
+        return user_meta_query.insert_user_meta_entries(db, metadata_ids)
 
-    except SQLAlchemyError as e:
-        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
-
-    return {"status": "success", "user_meta": [entry.id for entry in user_meta_entries]}
+        return handle_exception(e)
 
 
 @router.put("/book_collection")
 async def update_book_collection(
-    user: user_dependency,
-    db: db_dependency,
-    metadata_id: List[Optional[int]],  # Use the Pydantic model
+    user: user_dependency, db: db_dependency, metadata_ids: List[Optional[int]]
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    """Update user's book collection metadata."""
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
-        # Fetch and delete existing User_Meta entries
-        db.query(User_Meta).filter(User_Meta.user_id == user.get("id")).delete(
-            synchronize_session=False
-        )
-        db.commit()
-
-        # Insert new User_Meta entries
-        user_meta_entries = [
-            User_Meta(user_id=user.get("id"), metadata_id=mid) for mid in metadata_id
-        ]
-        db.add_all(user_meta_entries)
-        db.commit()
-    except SQLAlchemyError as e:
-        db.rollback()
-        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
-    except Exception as e:
-        return JSONResponse(status_code=500, content=f"Unexpected error: {str(e)}")
+        user_meta_query = UserMetaQuery(user)
+        return user_meta_query.update_user_meta_entries(db, metadata_ids)
 
-    return {"status": "success", "user_meta": [entry.id for entry in user_meta_entries]}
+    except Exception as e:
+        return handle_exception(e)
 
 
 @router.delete("/book_collection/{metadata_id}")
 async def delete_book_collection(
-    user: user_dependency,
-    db: db_dependency,
-    metadata_id: int
+    user: user_dependency, db: db_dependency, metadata_id: int
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    """Delete a specific book collection entry."""
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
-        # Query to delete the entry based on metadata_id for the authenticated user
-        entry_to_delete = db.query(User_Meta).filter(
-            User_Meta.user_id == user.get("id"),
-            User_Meta.metadata_id == metadata_id
-        ).first()
-
-        if not entry_to_delete:
-            return JSONResponse(status_code=404, content="Entry not found to delete.")
-
-        # Delete the entry
-        db.delete(entry_to_delete)
-        db.commit()
-
-    except SQLAlchemyError as e:
-        db.rollback()  # Rollback in case of any database error
-        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
-    except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
+        user_meta_query = UserMetaQuery(user)
+        return user_meta_query.delete_user_meta(
+            db, metadata_id=metadata_id
         )
-
-    return {
-        "status": "success",
-        "deleted_entry": entry_to_delete.id,
-    }
+    except Exception as e:
+        return handle_exception(e)
 
 
 @router.delete("/all_collections")
-async def delete_all_book(
-    user: user_dependency,
-    db: db_dependency,
-):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+async def delete_all_book(user: user_dependency, db: db_dependency):
+    """Delete all book collections for the authenticated user."""
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
-        db.query(User_Meta).filter(User_Meta.user_id == user.get("id")).delete(
-            synchronize_session="fetch"
-        )
-        
-        db.commit()  # Commit all deletions
+        user_meta_query = UserMetaQuery(user)
+        user_meta_query.delete_all_user_meta(db)
 
         return {
             "status": "success",
-            "delete book collection from": user.get("id"),
+            "message": f"Deleted book collection for user {user.get('id')}",
         }
-
-    except SQLAlchemyError as e:
-        db.rollback()  # Rollback in case of any database error
-        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
+        return handle_exception(e)

api/router/bot_general.py

@@ -1,19 +1,20 @@
+from typing import Annotated
+
 from fastapi import APIRouter, Depends
 from fastapi.responses import JSONResponse
-from service.dto import UserPromptRequest, BotResponse
+from sqlalchemy.orm import Session
+from sse_starlette.sse import EventSourceResponse
+
+from api.auth import check_user_authentication
+from api.function import generate_streaming_completion
+from api.router.user import user_dependency
+from core.chat.bot_service import ChatCompletionService
 from core.chat.chatstore import ChatStore
 from db.database import get_db
 from db.models import Session_Publisher
-from sqlalchemy.orm import Session
-from api.function import (
-    generate_streaming_completion,
-    generate_completion_non_streaming,
-)
-from api.router.user import user_dependency
-from sse_starlette.sse import EventSourceResponse
-from utils.utils import generate_uuid
-from typing import Annotated
 from langfuse.llama_index import LlamaIndexCallbackHandler
+from service.dto import UserPromptRequest, BotResponse
+from utils.utils import generate_uuid
 
 
 router = APIRouter(tags=["Bot_General"])
@@ -37,12 +38,16 @@ async def get_session_id(
     session_id: str,
     chat_store: ChatStore = Depends(get_chat_store),
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
+
+    # Retrieve chat history from Redis
+    chat_history = chat_store.get_messages(session_id)
 
-    chat_history = chat_store.get_all_messages_mongodb(session_id)
-    # if not chat_history:
-    #     return JSONResponse(status_code=404, content="Session not found or empty.")
+    # If no chat history is found in Redis, fallback to the alternative store using mongoDB
+    if chat_history is None or chat_history == []:
+        chat_history = chat_store.get_all_messages_mongodb(session_id)
 
     return chat_history
 
@@ -57,16 +62,11 @@ async def bot_generator_general(
 
     if user_prompt_request.streaming:
         return EventSourceResponse(
-            generate_streaming_completion(
-                user_prompt_request.prompt,
-                session_id
-            )
+            generate_streaming_completion(user_prompt_request.prompt, session_id)
         )
     else:
-        response, metadata, scores = generate_completion_non_streaming(
-            session_id,
-            user_prompt_request.prompt,
-        )
+        bot_service = ChatCompletionService(session_id, user_prompt_request.prompt)
+        response, metadata, scores = bot_service.generate_completion()
 
         return BotResponse(
             content=response,
@@ -76,12 +76,18 @@ async def bot_generator_general(
 
 
 @router.delete("/bot/{session_id}")
-async def delete_bot(db:db_dependency, session_id: str, chat_store: ChatStore = Depends(get_chat_store)):
+async def delete_bot(
+    db: db_dependency, session_id: str, chat_store: ChatStore = Depends(get_chat_store)
+):
     try:
         chat_store.delete_messages(session_id)
-        
         # Delete session from database
-        session = db.query(Session_Publisher).filter(Session_Publisher.id == session_id).first()
+        session = (
+            db.query(Session_Publisher)
+            .filter(Session_Publisher.id == session_id)
+            .first()
+        )
+
         if session:
             db.delete(session)
             db.commit()  # Commit the transaction

api/router/bot_one.py

@@ -1,24 +1,29 @@
 from typing import Annotated
 from fastapi import APIRouter, Depends
 from fastapi.responses import JSONResponse
-from service.dto import UserPromptRequest, BotResponse
+import pytz
+
+from service.dto import UserPromptRequest, BotResponse, BotCreateRequest
+from datetime import datetime
 from core.chat.chatstore import ChatStore
+from core.chat.bot_service import ChatCompletionService
 from db.database import get_db
-from db.models import Metadata, Session_Publisher
-from db.models import Session as SessionModel
+from db.models import Session_Publisher
+from db.query.query_book import BookQuery
+from db.query.query_bot import BotQuery
+
 from sqlalchemy.orm import Session
-from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy import select
-from api.function import (
-    generate_streaming_completion,
-    generate_completion_non_streaming,
-)
-from datetime import datetime
-from api.router.user import user_dependency
 from sse_starlette.sse import EventSourceResponse
 from utils.utils import generate_uuid
+from utils.error_handlers import handle_exception
 from langfuse.llama_index import LlamaIndexCallbackHandler
 
+from api.auth import check_user_authentication
+from api.router.user import user_dependency
+from api.function import (
+    generate_streaming_completion
+)
+
 
 router = APIRouter(tags=["Bot_One"])
 
@@ -30,25 +35,20 @@ def get_chat_store():
 
 
 @router.post("/bot_one/{metadata_id}")
-async def create_bot_one(user: user_dependency, db: db_dependency, metadata_id: int):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+async def create_bot_one(user: user_dependency, db: db_dependency, metadata_id: int, bot_name:BotCreateRequest):
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
     # Generate a new session ID (UUID)
     try:
         session_id = generate_uuid()
-
-        # Create the new session
-        new_session = Session_Publisher(
-            id=session_id,
-            user_id=user.get("id"),
-            metadata_id=metadata_id,
-        )
-
-        db.add(new_session)
-        db.commit()  # Commit the new session to the database
+        
+        bot_query = BotQuery(user)
+        bot_query.add_bot(db, session_id, bot_name.name, metadata_id)
 
         return {
-            "statur": "session id created successfully",
+            "status": "session id created successfully",
+            "bot_name": bot_name.name,
             "session_id": session_id,
         }
 
@@ -59,15 +59,16 @@ async def create_bot_one(user: user_dependency, db: db_dependency, metadata_id:
 
 
 @router.post("/bot_one/{metadata_id}/{session_id}")
-async def generator_bot_one(
+async def generator_bot(
     user: user_dependency,
     db: db_dependency,
     metadata_id: int,
     session_id: str,
     user_prompt_request: UserPromptRequest,
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     langfuse_callback_handler = LlamaIndexCallbackHandler()
     langfuse_callback_handler.set_trace_params(
@@ -76,25 +77,13 @@ async def generator_bot_one(
 
     # Query to retrieve the titles
     try:
-        query = (
-            select(Metadata.title)
-            .join(Session_Publisher, Metadata.id == metadata_id)
-            .where(
-                Session_Publisher.user_id == user.get("id"),
-                Session_Publisher.id == session_id,
-            )
-        )
-
-        result = db.execute(query)
-        titles = result.scalars().all()
+        book_query = BookQuery(user)
+        output_titles = book_query.get_title_from_session(db, metadata_id, session_id)
+        titles = [item[0] for item in output_titles]
         print(titles)
 
-    except SQLAlchemyError as e:
-        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
+        return handle_exception(e)
 
     if user_prompt_request.streaming:
         return EventSourceResponse(
@@ -104,14 +93,17 @@ async def generator_bot_one(
             )
         )
     else:
-        response, metadata, scores = generate_completion_non_streaming(
-            session_id, user_prompt_request.prompt, titles, type_bot="specific"
-        )
+        bot_service = ChatCompletionService(session_id, user_prompt_request.prompt, titles, type_bot="specific")
+        response, metadata, scores = bot_service.generate_completion()
+
+        # Set Jakarta timezone
+        jakarta_tz = pytz.timezone('Asia/Jakarta')
 
         existing_session = (
             db.query(Session_Publisher).filter(Session_Publisher.id == session_id).first()
         )
-        existing_session.updated_at = datetime.now()
+        
+        existing_session.updated_at = datetime.now(jakarta_tz)
         db.commit()
         
         return BotResponse(
@@ -121,24 +113,20 @@ async def generator_bot_one(
         )
 
 
-@router.get("/bot_one{metadata_id}")
+@router.get("/bot_one/{metadata_id}")
 async def get_all_session_bot_one(
     user: user_dependency, db: db_dependency, metadata_id: int
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
         # Query the session IDs based on the user ID
-        query = select(Session_Publisher.id, Session_Publisher.updated_at).where(
-            Session_Publisher.user_id == user.get("id"),
-            Session_Publisher.metadata_id == metadata_id,
-        )
-
-        result = db.execute(query)
-        sessions = result.all()
+        bot_query = BotQuery(user)
+        sessions = bot_query.get_session_ids_bot(db, metadata_id)
         
-        session_data = [{"id": session.id, "updated_at": str(session.updated_at)} for session in sessions]
+        session_data = [{"id": session.id, "bot_name":session.bot_name, "updated_at": str(session.updated_at)} for session in sessions]
 
         # Convert list of tuples to a simple list
         session_sorted_data = sorted(session_data, key=lambda x: datetime.fromisoformat(x['updated_at']), reverse=True)

api/router/bot_specific.py

@@ -1,24 +1,25 @@
 from typing import Annotated, List, Optional
+from datetime import datetime
+
 from fastapi import APIRouter, Depends
 from fastapi.responses import JSONResponse
+from sse_starlette.sse import EventSourceResponse
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from sqlalchemy.exc import SQLAlchemyError, NoResultFound
+
 from service.dto import UserPromptRequest, BotResponse, BotCreateRequest
 from core.chat.chatstore import ChatStore
+from core.chat.bot_service import ChatCompletionService
 from db.database import get_db
 from db.models import Bot_Meta, Bot, Metadata
 from db.models import Session as SessionModel
-from sqlalchemy.orm import Session
-from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy.exc import NoResultFound
-from sqlalchemy import select
-from api.function import (
-    generate_streaming_completion,
-    generate_completion_non_streaming,
-)
+from langfuse.llama_index import LlamaIndexCallbackHandler
+from api.auth import check_user_authentication
 from api.router.user import user_dependency
-from sse_starlette.sse import EventSourceResponse
+from api.function import generate_streaming_completion
 from utils.utils import generate_uuid
-from langfuse.llama_index import LlamaIndexCallbackHandler
-from datetime import datetime
+from utils.error_handlers import handle_exception
 
 
 router = APIRouter(tags=["Bot_Specific"])
@@ -37,14 +38,15 @@ async def create_bot_id(
     bot_request: BotCreateRequest,
 ):
 
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     # Create a new bot entry
     try:
         # Create a new bot entry
         new_bot = Bot(
-            user_id=user.get("id"), bot_name=bot_request.bot_name
+            user_id=user.get("id"), bot_name=bot_request.name
         )  # Assuming user has an 'id' attribute
 
         db.add(new_bot)
@@ -69,8 +71,9 @@ async def create_bot_specific(
     bot_id: int,
     metadata_id: List[Optional[int]],  # Use the Pydantic model
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
         # Create BotMeta instances for each metadata_id
@@ -99,8 +102,9 @@ async def update_bot_specific(
     bot_id: int,
     metadata_id: List[Optional[int]],  # Use the Pydantic model
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
         # Fetch existing Bot_Meta entries related to bot_id
@@ -119,16 +123,16 @@ async def update_bot_specific(
         ]
         db.add_all(bot_meta_entries)
         db.commit()
+        return {
+            "status": "success",
+            "bot_meta": [entry.id for entry in bot_meta_entries],
+        }
 
     except SQLAlchemyError as e:
         db.rollback()  # Rollback in case of any database error
         return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
-
-    return {"status": "success", "bot_meta": [entry.id for entry in bot_meta_entries]}
+        return handle_exception(e)
 
 
 @router.delete("/meta/{bot_id}/{metadata_id}")
@@ -138,8 +142,9 @@ async def delete_bot_specific(
     bot_id: int,
     metadata_id: int,  # Changed to int to specify a single metadata_id
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
         # Delete the specific metadata entry for the given bot_id
@@ -155,19 +160,16 @@ async def delete_bot_specific(
         # Delete the found entry
         db.delete(bot_meta_entry)
         db.commit()
+        return {
+            "status": "success",
+            "deleted_entry_id": bot_meta_entry.id,
+        }
 
     except SQLAlchemyError as e:
         db.rollback()  # Rollback in case of any database error
         return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
-
-    return {
-        "status": "success",
-        "deleted_entry_id": bot_meta_entry.id,
-    }
+        return handle_exception(e)
 
 
 @router.delete("/bot_all/{bot_id}")
@@ -176,8 +178,9 @@ async def delete_bot_id(
     db: db_dependency,
     bot_id: int,
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
         # Fetch the bot to ensure it exists
@@ -207,18 +210,16 @@ async def delete_bot_id(
         db.rollback()  # Rollback in case of any database error
         return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
+        return handle_exception(e)
 
 
 @router.post("/session/{bot_id}/new")
 async def create_new_session(user: user_dependency, db: db_dependency, bot_id: int):
     # Check if user is authenticated
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
-    print(user.get("id"))
     user_id = user.get("id")
     # Ensure the bot belongs to the user
     bot_query = select(Bot).where(Bot.id == bot_id, Bot.user_id == user_id)
@@ -251,32 +252,30 @@ async def create_new_session(user: user_dependency, db: db_dependency, bot_id: i
         }
 
     except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected  in retrieving session id {str(e)}"
-        )
+        return handle_exception(e)
 
 
 @router.get("/bot/all/{bot_id}")
 async def get_all_session_ids(user: user_dependency, db: db_dependency, bot_id: int):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
 
     try:
         query = select(SessionModel.id, SessionModel.updated_at).where(
-            SessionModel.user_id == user.get("id"),
-            SessionModel.bot_id == bot_id
+            SessionModel.user_id == user.get("id"), SessionModel.bot_id == bot_id
         )
 
         result = db.execute(query)
         sessions = result.all()
-        session_data = [{"id": session.id, "updated_at": session.updated_at} for session in sessions]
+        session_data = [
+            {"id": session.id, "updated_at": session.updated_at} for session in sessions
+        ]
 
         return session_data
 
     except Exception as e:
-        # Log the error and return JSONResponse for FastAPI
-        print(f"An error occurred while fetching session IDs: {e}")
-        return JSONResponse(status_code=400, content="Error retrieving session IDs")
+        return handle_exception(e)
 
 
 @router.post("/bot/{bot_id}/{session_id}")
@@ -287,9 +286,9 @@ async def bot_generator_spesific(
     session_id: str,
     user_prompt_request: UserPromptRequest,
 ):
-    if user is None:
-        return JSONResponse(status_code=401, content="Authentication Failed")
-
+    auth_response = check_user_authentication(user)
+    if auth_response:
+        return auth_response
     langfuse_callback_handler = LlamaIndexCallbackHandler()
     langfuse_callback_handler.set_trace_params(
         user_id=user.get("username"), session_id=session_id
@@ -310,33 +309,30 @@ async def bot_generator_spesific(
         titles = result.scalars().all()
         print(titles)
 
-    except SQLAlchemyError as e:
-        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
-    except Exception as e:
-        return JSONResponse(
-            status_code=500, content=f"An unexpected error occurred: {str(e)}"
-        )
+        if user_prompt_request.streaming:
+            return EventSourceResponse(
+                generate_streaming_completion(
+                    user_prompt_request.prompt,
+                    session_id,
+                )
+            )
+        else:
+            bot_service = ChatCompletionService(session_id, user_prompt_request.prompt, titles, type_bot="specific")
+            response, metadata, scores = bot_service.generate_completion()
 
-    if user_prompt_request.streaming:
-        return EventSourceResponse(
-            generate_streaming_completion(
-                user_prompt_request.prompt,
-                session_id,
+            existing_session = (
+                db.query(SessionModel).filter(SessionModel.id == session_id).first()
             )
-        )
-    else:
-        response, metadata, scores = generate_completion_non_streaming(
-            session_id, user_prompt_request.prompt, titles, type_bot="specific"
-        )
-        
-        existing_session = (
-            db.query(SessionModel).filter(SessionModel.id == session_id).first()
-        )
-        existing_session.updated_at = datetime.now()
-        db.commit()
+            existing_session.updated_at = datetime.now()
+            db.commit()
 
-        return BotResponse(
-            content=response,
-            metadata=metadata,
-            scores=scores,
-        )
+            return BotResponse(
+                content=response,
+                metadata=metadata,
+                scores=scores,
+            )
+
+    except SQLAlchemyError as e:
+        return JSONResponse(status_code=500, content=f"Database error: {str(e)}")
+    except Exception as e:
+        return handle_exception(e)

api/router/category.py

@@ -1,6 +1,6 @@
 from api.router.user import user_dependency
 from fastapi.responses import JSONResponse
-from fastapi import APIRouter, HTTPException, Depends, Query
+from fastapi import APIRouter, Depends
 
 from db.models import Category
 from db.database import get_db

api/router/role.py

@@ -1,4 +1,4 @@
-from fastapi import APIRouter, HTTPException, Depends
+from fastapi import APIRouter, Depends
 from fastapi.responses import JSONResponse
 from api.router.user import user_dependency
 from typing import Annotated

api/router/user.py

@@ -1,14 +1,16 @@
+from datetime import timedelta
+from typing import Annotated
+
 from fastapi import APIRouter, Depends, status
-from fastapi.security import OAuth2PasswordRequestForm
 from fastapi.responses import JSONResponse
+from fastapi.security import OAuth2PasswordRequestForm
+from passlib.context import CryptContext
+from sqlalchemy.orm import Session
+
 from db.models import User
 from db.database import get_db
 from api.auth import get_current_user, create_access_token
 from service.dto import CreateUserRequest, UserVerification, Token
-from typing import Annotated
-from passlib.context import CryptContext
-from sqlalchemy.orm import Session
-from datetime import timedelta
 
 
 router = APIRouter(tags=["User"])
@@ -38,27 +40,28 @@ async def login_for_access_token(
     try:
         access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
         access_token = create_access_token(
-            user.username, user.name, user.id, user.role_id, access_token_expires, user.email
+            user.username,
+            user.name,
+            user.id,
+            user.role_id,
+            access_token_expires,
+            user.email,
         )
 
         return {"access_token": access_token, "token_type": "bearer"}
 
     except Exception as e:
         print(e)
-        return JSONResponse(
-            status_code=500, content="An error occuring when login"
-        )
+        return JSONResponse(status_code=500, content="An error occuring when login")
 
 
 @router.get("/login", response_model=dict)
 async def get_user(user: user_dependency):
     if user is None:
-        return JSONResponse(
-            status_code=401, content="Authentication Failed"
-        )
+        return JSONResponse(status_code=401, content="Authentication Failed")
     return {
         "username": user.get("username"),
-        "name" : user.get("name"),
+        "name": user.get("name"),
         "id": user.get("id"),
         "email": user.get("email"),
         "role": user.get("role_id"),
@@ -69,9 +72,7 @@ async def get_user(user: user_dependency):
 async def get_all_users(user: user_dependency, db: Session = Depends(get_db)):
     # Check if the current user has an admin role
     if user.get("role_id") != 1:  # Adjust this check based on how roles are represented
-        return JSONResponse(
-            status_code=401, content="Authentication Failed"
-        )
+        return JSONResponse(status_code=401, content="Authentication Failed")
 
     # Query the database to retrieve all users
     users = db.query(
@@ -96,9 +97,7 @@ async def register_user(db: db_dependency, create_user_request: CreateUserReques
     )
 
     if existing_user:
-        return JSONResponse(
-            status_code=400, content="Email is already registered"
-        )
+        return JSONResponse(status_code=400, content="Email is already registered")
 
     try:
         password_hash = bcrypt_context.hash(create_user_request.password)
@@ -133,21 +132,17 @@ async def change_password(
     user: user_dependency, db: db_dependency, user_verification: UserVerification
 ):
     if user is None:
-        return JSONResponse(
-            status_code=401, content="Authentication Failed"
-        )
+        return JSONResponse(status_code=401, content="Authentication Failed")
     user_model = db.query(User).filter(User.id == user.get("id")).first()
 
     if not bcrypt_context.verify(
         user_verification.password, user_model.hashed_password
     ):
-        return JSONResponse(
-            status_code=401, content="Error on password change"
-        )
+        return JSONResponse(status_code=401, content="Error on password change")
 
     user_model.hashed_password = bcrypt_context.hash(user_verification.new_password)
     db.add(user_model)
     db.commit()
     db.refresh(user_model)
 
-    return {"message": "User's password successfully changed", "user_id": user_model.id}
+    return {"message": "User's password successfully changed", "user_id": user_model.id}

config.py

@@ -1,6 +1,8 @@
 from pydantic_settings import BaseSettings
+from dotenv import load_dotenv
 import os
 
+load_dotenv()
 
 class MysqlConfig(BaseSettings):
     DB_HOST: str = ""

core/chat/bot_service.py

@@ -0,0 +1,163 @@
+import logging
+import re
+import os
+
+from typing import List
+from datetime import datetime
+from fastapi.responses import JSONResponse
+from script.vector_db import IndexManager
+from llama_index.core.llms import MessageRole
+
+from core.chat.engine import Engine
+from core.chat.chatstore import ChatStore
+from core.parser import clean_text, update_response, renumber_sources
+
+from service.dto import ChatMessage
+from pymongo.mongo_client import MongoClient
+
+
+
+
+class ChatCompletionService:
+    def __init__(self, session_id: str, user_request: str, titles: List = None, type_bot: str = "general"):
+        self.session_id = session_id
+        self.user_request = user_request
+        self.titles = titles
+        self.type_bot = type_bot
+        self.client = MongoClient(os.getenv("MONGO_URI"))
+        self.engine = Engine()
+        self.index_manager = IndexManager()
+        self.chatstore = ChatStore()
+
+    def generate_completion(self):
+        if not self._ping_mongo():
+            return JSONResponse(status_code=500, content="Database Error: Unable to connect to MongoDB")
+
+        try:
+            # Load and retrieve chat engine with appropriate index
+            index = self.index_manager.load_existing_indexes()
+            chat_engine = self._get_chat_engine(index)
+
+            # Generate chat response
+            response = chat_engine.chat(self.user_request)
+            sources = response.sources
+            number_reference_sorted = self._extract_sorted_references(response)
+
+            contents, metadata_collection, scores = self._process_sources(sources, number_reference_sorted)
+
+            # Update response and renumber sources
+            response = update_response(str(response))
+            contents = renumber_sources(contents)
+
+            # Add contents to metadata
+            metadata_collection = self._attach_contents_to_metadata(contents, metadata_collection)
+
+            # Save the message to chat store
+            self._store_message_in_chatstore(response, metadata_collection)
+
+        except Exception as e:
+            logging.error(f"An error occurred in generate text: {e}")
+            return JSONResponse(
+                status_code=500,
+                content=f"An internal server error occurred: {e}"
+            )
+
+        try:
+            if self.type_bot == "specific":
+                self._save_chat_history_to_db(response, metadata_collection)
+
+            return str(response), metadata_collection, scores
+
+        except Exception as e:
+            logging.error(f"An error occurred while saving chat history: {e}")
+            return JSONResponse(
+                status_code=500,
+                content=f"An internal server error occurred while saving chat history: {e}"
+            )
+
+    def _ping_mongo(self):
+        try:
+            self.client.admin.command("ping")
+            print("Pinged your deployment. Successfully connected to MongoDB!")
+            return True
+        except Exception as e:
+            logging.error(f"MongoDB connection failed: {e}")
+            return False
+
+    def _get_chat_engine(self, index):
+        if self.type_bot == "general":
+            return self.engine.get_chat_engine(self.session_id, index)
+        return self.engine.get_chat_engine(self.session_id, index, self.titles, self.type_bot)
+
+    def _extract_sorted_references(self, response):
+        number_reference = list(set(re.findall(r"\[(\d+)\]", str(response))))
+        return sorted(number_reference)
+
+    def _process_sources(self, sources, number_reference_sorted):
+        contents, metadata_collection, scores = [], [], []
+        if not number_reference_sorted:
+            print("There are no references")
+            return contents, metadata_collection, scores
+
+        for number in number_reference_sorted:
+            number = int(number)
+            if sources and sources[0].get("raw_output"):
+                node = dict(sources[0])["raw_output"].source_nodes
+                if 0 <= number - 1 < len(node):
+                    content = clean_text(node[number - 1].node.get_text())
+                    contents.append(content)
+                    metadata = dict(node[number - 1].node.metadata)
+                    metadata_collection.append(metadata)
+                    score = node[number - 1].score
+                    scores.append(score)
+                else:
+                    print(f"Invalid reference number: {number}")
+            else:
+                print("No sources available")
+
+        return contents, metadata_collection, scores
+
+    def _attach_contents_to_metadata(self, contents, metadata_collection):
+        for i in range(min(len(contents), len(metadata_collection))):
+            metadata_collection[i]["content"] = re.sub(r"source \d+:", "", contents[i])
+        return metadata_collection
+
+    def _store_message_in_chatstore(self, response, metadata_collection):
+        message = ChatMessage(
+            role=MessageRole.ASSISTANT,
+            content=response,
+            metadata=metadata_collection
+        )
+        self.chatstore.delete_last_message(self.session_id)
+        self.chatstore.add_message(self.session_id, message)
+        self.chatstore.clean_message(self.session_id)
+
+    def _save_chat_history_to_db(self, response, metadata_collection):
+        chat_history_db = [
+            ChatMessage(
+                role=MessageRole.SYSTEM,
+                content=self.user_request,
+                timestamp=datetime.now(),
+                payment="free" if self.type_bot == "general" else None,
+            ),
+            ChatMessage(
+                role=MessageRole.ASSISTANT,
+                content=response,
+                metadata=metadata_collection,
+                timestamp=datetime.now(),
+                payment="free" if self.type_bot == "general" else None,
+            ),
+        ]
+
+        chat_history_json = [message.model_dump() for message in chat_history_db]
+
+        db = self.client["bot_database"]  # Replace with your database name
+        collection = db[self.session_id]  # Replace with your collection name
+        result = collection.insert_many(chat_history_json)
+        print("Data inserted with record ids", result.inserted_ids)
+
+
+# Example usage
+def generate_completion_non_streaming(session_id, user_request, titles=None, type_bot="general"):
+    chat_service = ChatCompletionService(session_id, user_request, titles, type_bot)
+    return chat_service.generate_completion()

core/chat/chatstore.py

@@ -2,7 +2,7 @@ import redis
 import os
 import json
 from fastapi.responses import JSONResponse
-from typing import Optional, List
+from typing import Optional, List, Dict
 from llama_index.storage.chat_store.redis import RedisChatStore
 from pymongo.mongo_client import MongoClient
 from llama_index.core.memory import ChatMemoryBuffer
@@ -56,6 +56,17 @@ class ChatStore:
 
         # Decode and parse each item into a dictionary
         return [json.loads(m.decode("utf-8")) for m in items]
+    
+    def get_last_message(self, session_id: str) -> Optional[Dict]:
+        """Get the last message for a session_id."""
+        last_message = self.redis_client.lindex(session_id, -1)
+        
+        if last_message is None:
+            return None  # Return None if there are no messages
+        
+        # Decode and parse the last message into a dictionary
+        return json.loads(last_message.decode("utf-8"))
+
 
     def delete_last_message(self, session_id: str) -> Optional[ChatMessage]:
         """Delete last message for a session_id."""

core/prompt.py

@@ -1,5 +1,5 @@
 SYSTEM_BOT_TEMPLATE = """
-Kamu adalah Medbot yang selalu menggunakan tools kamu untuk menjawab pertanyaan tentang kedokteran. Tugasmu adalah memberikan jawaban yang informatif dan akurat berdasarkan tools yang tersedia. {additional_information} Jika setelah itu tidak ada informasi yang ditemukan, katakan bahwa kamu tidak mengetahuinya dan berikan informasi dari apa yang kamu ketahui kemudian arahkan pengguna untuk bertanya ke dokter yang lebih ahli.
+Kamu adalah Medbot yang selalu menggunakan tools untuk menjawab pertanyaan medis. Jika pengguna bertanya tentang topik non-medis, arahkan mereka untuk bertanya di bidang medis. Tugasmu adalah memberikan jawaban yang informatif dan akurat berdasarkan tools yang tersedia. Pastikan kamu hanya memberikan informasi dari buku yang telah disediakan, jangan sampai menjawab pertanyaan yang tidak terdapat dalam buku atau tools yang kamu gunakan. {additional_information} Jika setelah itu tidak ada informasi yang ditemukan, katakan bahwa kamu tidak mengetahuinya dan berikan informasi dari apa yang kamu ketahui kemudian arahkan pengguna untuk bertanya ke dokter yang lebih ahli. 
 
 **Instruksi**:
 

db/database.py

@@ -2,39 +2,45 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.exc import OperationalError
 from config import MYSQL_CONFIG
-from fastapi import HTTPException
+from fastapi.responses import JSONResponse
+from fastapi import HTTPException, status
+from dotenv import load_dotenv
+import io
 import os
 import base64
 
+load_dotenv()
 
 SQLALCHEMY_DATABASE_URL = MYSQL_CONFIG.DB_URI_SQL_ALCHEMY
 
 # Get the base64 encoded certificate from the environment variable
 ca_cert_base64 = os.getenv("CA_CERT_BASE64")
 
-# Decode the base64 content
+# Retrieve the Base64-encoded CA certificate from the environment variable
+ca_cert_base64 = os.getenv("CA_CERT_BASE64")
+
 if ca_cert_base64:
+    # Decode the base64 content
     ca_cert_content = base64.b64decode(ca_cert_base64).decode("utf-8")
 
-    # Write the decoded content to a temporary .pem file
-    with open("/tmp/ca.pem", "w") as f:
-        f.write(ca_cert_content)
+    # Use in-memory buffer to handle the decoded CA certificate content
+    ca_cert_in_memory = io.StringIO(ca_cert_content)
+
+    # Create SQLAlchemy engine with SSL configuration, using in-memory certificate
+    engine = create_engine(
+        SQLALCHEMY_DATABASE_URL,
+        connect_args={
+            "ssl": {
+                "sslmode": "REQUIRED",
+                "sslrootcert": ca_cert_in_memory,  # In-memory CA certificate
+                # Add other SSL options like client cert/key if required
+            }
+        },
+    )
 
-    ca_cert_path = "/tmp/ca.pem"
 else:
     raise ValueError("CA_CERT_BASE64 environment variable is not set")
 
-# Use the decoded CA certificate in the SQLAlchemy engine
-engine = create_engine(
-    SQLALCHEMY_DATABASE_URL,
-    connect_args={
-        "ssl": {
-            "sslmode": "REQUIRED",
-            "ca": ca_cert_path,  # Path to the temporary CA certificate
-            # Add other SSL options as needed
-        }
-    },
-)
 
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 

db/fetching.py

@@ -0,0 +1,60 @@
+from db.query.query_user_meta import UserMetaQuery
+from db.query.query_book import BookQuery
+from service.dto import MetadataResponse
+from fastapi.responses import JSONResponse
+
+class DataFetching:
+    def __init__(self, user, db):
+        self.user = user
+        self.db = db
+        
+    def collection_fetching(self):
+        user_meta_query = UserMetaQuery(self.user)
+        user_meta_entries = user_meta_query.get_user_meta_entries(self.db)
+        
+        if not user_meta_entries:
+            return {"info": "No book collection found"}
+        
+        # Extract relevant data from the user_meta_entries
+        
+        
+        results = [
+            {
+                "user_id": user_meta.user_id,  # Ensure you include user_id if needed
+                "metadata_id": metadata.id,
+                "title": metadata.title,
+                "author": metadata.author,
+                "category_name": category.category,
+                "year": metadata.year,
+                "publisher": metadata.publisher,
+            }
+            for user_meta, metadata, category in user_meta_entries  # Unpack the tuple
+        ]
+        
+        print("Hasil akhir ", results)
+        
+        # Extract relevant data from the user_meta_entries
+        return results
+    
+    def metadata_fetching(self):
+        book_query = BookQuery(self.user)
+        book_query_entries = book_query.get_book(self.db)
+        
+        return [
+            MetadataResponse(
+                id = id,
+                title=title,
+                author=author,
+                category=category,
+                category_id = category_id,
+                year=year,
+                publisher=publisher,
+                thumbnail=(
+                    thumbnail if thumbnail else None
+                ),  # Ensure None if thumbnail is not present
+            )
+            for id, title, author, category, category_id, year, publisher, thumbnail in book_query_entries
+        ]
+        
+        
+    

db/models.py

@@ -159,6 +159,7 @@ class Session_Publisher(Base):
     
     id = mapped_column(String(36), primary_key=True, index=True, default=lambda: str(uuid.uuid4()))  # Store as string
     user_id = mapped_column(Integer, ForeignKey("user.id"))
+    bot_name = mapped_column(String(100), nullable=True)
     metadata_id = mapped_column(Integer, ForeignKey("metadata.id"))
     created_at : Mapped[timestamp_current] 
     updated_at : Mapped[timestamp_update]

db/query/base_query.py

@@ -0,0 +1,201 @@
+from sqlalchemy import select, delete, update
+from sqlalchemy.exc import SQLAlchemyError
+from utils.error_handlers import handle_error, not_found_error, no_entries_found, handle_exception
+from fastapi.responses import JSONResponse
+from typing import List, Type, Optional
+
+
+class BaseQuery:
+    def __init__(self, user):
+        self.user = user
+        self.user_id = user.get("id")
+
+    def _fetch(self, db, query, not_found_message, multiple: bool = False):
+        """Fetch a single or multiple results based on the 'multiple' flag."""
+        try:
+            if multiple:
+                results = db.execute(query).all()
+                if not results:
+                    return no_entries_found(not_found_message)
+                return results
+            else:
+                result = db.execute(query).scalar_one_or_none()
+                if not result:
+                    return not_found_error(not_found_message)
+                return result
+        except Exception as e:
+            return handle_error(
+                e,
+                "Failed to fetch entry" if not multiple else "Failed to fetch entries",
+            )
+
+    def _handle_commit(self, db):
+        try:
+            db.commit()
+        except SQLAlchemyError as e:
+            db.rollback()
+            return handle_exception(e)
+        except Exception as e:
+            db.rollback()
+            return handle_error(e, "Operation failed")
+
+    def add(self, db, instance):
+        """Add a new entry."""
+        db.add(instance)
+        return self._handle_commit(db)
+
+    def insert_entries(self, db, entries):
+        """Insert multiple entries."""
+        db.add_all(entries)
+        return self._handle_commit(db)
+
+    def delete(self, db, model, id, filter_conditions=None):
+        """Delete an entry by ID with optional filter conditions."""
+        # Build the query to select the entry
+        query = select(model).where(model.id == id)
+        if filter_conditions:
+            query = query.where(*filter_conditions)
+        
+        # Fetch the entry
+        entry = self._fetch(db, query, f"Entry with ID {id} not found.", multiple=False)
+        if isinstance(entry, JSONResponse):
+            return entry
+
+        # Build the delete query
+        delete_query = delete(model).where(model.id == id)
+        if filter_conditions:
+            delete_query = delete_query.where(*filter_conditions)
+        
+        # Execute the delete query and commit
+        db.execute(delete_query)
+        return self._handle_commit(db)
+
+
+    def delete_all(self, db, model, filter_conditions=None):
+        """Delete all entries or based on filters."""
+        query = delete(model)
+        if filter_conditions:
+            query = query.where(*filter_conditions)
+        db.execute(query)
+        return self._handle_commit(db)
+
+    def update(self, db, model, id, update_data, filter_conditions=None):
+        """Update an entry by ID."""
+        query = select(model).where(model.id == id)
+        if filter_conditions:
+            query = query.where(*filter_conditions)
+            
+        not_found_message = f"Entry with ID {id} not found."    
+        entry = self._fetch(db, query, not_found_message, multiple=False)
+        
+        if isinstance(entry, JSONResponse):
+            return entry
+
+        db.execute(
+            update(model)
+            .where(model.id == id, model.user_id == self.user_id)
+            .values(update_data)
+        )
+        return self._handle_commit(db)
+
+    def update_entries(self, db, model, update_data, filter_conditions=None):
+        """Update multiple entries with optional filtering."""
+        query = select(model)
+        if filter_conditions:
+            query = query.where(*filter_conditions)
+        not_found_message = "No entries found matching the filter conditions."
+        results = self._fetch(
+            db, query, not_found_message , multiple=True
+        )
+        if isinstance(results, JSONResponse):
+            return results
+
+        db.execute(update(model).where(*filter_conditions).values(update_data))
+        return self._handle_commit(db)
+
+    def get(
+        self,
+        db,
+        model: Type = None,
+        id: Optional[int] = None,
+        filter_conditions=None,
+        columns: Optional[List[str]] = None,
+        multiple: bool = False,
+    ):
+        """Get one or multiple entries, filtered by ID or conditions."""
+        if columns:
+            query = select(*columns)
+        else:
+            query = select(model)
+
+        # Apply filtering by user ID and optional conditions
+        query = query
+        if id:
+            query = query.where(model.id == id)
+        if filter_conditions:
+            query = query.where(*filter_conditions)
+
+        return self._fetch(
+            db,
+            query,
+            "Entry not found." if not multiple else "No entries found.",
+            multiple=multiple,
+        )
+
+    def get_with_joins(
+        self,
+        db,
+        join_models: List[Type],
+        join_conditions: List=None,
+        model: Type=None,
+        filter_conditions=None,
+        columns: Optional[List[str]] = None,
+        multiple: bool = False,
+    ):
+        """Get one or multiple entries with joins and optional filters."""
+        if columns:
+            query = select(*columns)
+        else:
+            query = select(model, *join_models).select_from(model)
+
+        # Apply joins
+        if join_conditions:
+            for join_model, join_condition in zip(join_models, join_conditions):
+                query = query.join(join_model, join_condition)
+        else: 
+            query = query.join(*join_models)
+
+        # Apply filtering by user ID and optional conditions
+        if filter_conditions:
+            query = query.where(*filter_conditions)
+
+        return self._fetch(
+            db,
+            query,
+            "Entry not found." if not multiple else "No entries found.",
+            multiple=multiple,
+        )
+
+    def get_columns(
+        self,
+        db,
+        columns: List[str],
+        model=None,
+        filter_conditions=None,
+        id: Optional[int] = None,
+        multiple: bool = False,
+    ):
+        """Get specific columns by ID or filtering."""
+        query = select(*columns).select_from(model)
+
+        if id:
+            query = query.where(model.id == id)
+        if filter_conditions:
+            query = query.where(*filter_conditions)
+
+        return self._fetch(
+            db,
+            query,
+            "Entry not found." if not multiple else "No entries found.",
+            multiple=multiple,
+        )

db/query/query_book.py

@@ -0,0 +1,124 @@
+from db.query.base_query import BaseQuery
+from db.models import Metadata, Category, Session_Publisher
+from fastapi.responses import JSONResponse
+
+
+class BookQuery(BaseQuery):
+    def __init__(self, user):
+        super().__init__(user)
+
+    def add_book(self, db, title, author, category_id, year, publisher):
+        new_book = Metadata(
+            title=title,
+            author=author,
+            category_id=category_id,
+            year=year,
+            publisher=publisher,
+        )
+        self.add(db, new_book)
+
+    def get_book(self, db):
+        model = Metadata
+        metadata_columns = [
+            Metadata.id,
+            Metadata.title,
+            Metadata.author,
+            Category.category,  # Assuming this is the correct field for category name
+            Category.id,
+            Metadata.year,
+            Metadata.publisher,
+            Metadata.thumbnail,
+        ]
+
+        join_models = [Category]
+        join_conditions = [Metadata.category_id == Category.id]
+
+        result = self.get_with_joins(
+            db,
+            model=model,
+            columns=metadata_columns,
+            join_models=join_models,
+            join_conditions=join_conditions,
+            multiple=True,
+        )
+        print("result", result)
+
+        return result
+
+    def update_metadata_entry(
+        self, db, metadata_id, title, author, category_id, year, publisher
+    ):
+        model = Metadata
+        # Define filter conditions based on the metadata_id
+        filter_conditions = [Metadata.id == metadata_id]
+
+        # Prepare the update data
+        update_data = {
+            "title": title,
+            "author": author,
+            "category_id": category_id,
+            "year": year,
+            "publisher": publisher,
+        }
+
+        # Call the update_entries method to update the metadata
+        update_response = self.update_entries(
+            db,
+            model=model,
+            update_data=update_data,
+            filter_conditions=filter_conditions,
+        )
+
+        if isinstance(update_response, JSONResponse):
+            return update_response  # Return error response if any
+
+        # Fetch the updated metadata to retrieve the new category
+        updated_metadata = db.query(Metadata).filter(Metadata.id == metadata_id).first()
+        if not updated_metadata:
+            return JSONResponse(status_code=404, content="Metadata not found")
+
+        return updated_metadata
+
+    def update_book(self, db, book_id, title, author):
+        update_data = {"title": title, "author": author}
+        self.update(db, Metadata, book_id, update_data)
+
+    def delete_book(self, db, book_id):
+        self.delete(db, Metadata, book_id)
+
+    def get_books(self, db):
+        return self.get(db, model=Metadata, multiple=True)
+
+    def get_metadata_books(self, db, metadata_id):
+        return self.get(db, Metadata, id=metadata_id)
+
+    # def get_title_from_session(self, db, metadata_id, session_id):
+    #     model = Session_Publisher
+    #     columns = [Metadata.title]
+    #     join_models = [Session_Publisher.id == session_id, Metadata.id == metadata_id]
+
+    #     titles = self.get_all_with_join_columns(db, model, columns, join_models)
+
+    #     return titles
+
+    def get_title_from_session(self, db, metadata_id, session_id):
+        model = Session_Publisher
+        columns = [Metadata.title]
+        join_models = [Session_Publisher]
+        join_conditions = [Metadata.id == metadata_id]
+        filter_conditions = [
+            Session_Publisher.user_id == self.user_id,
+            Session_Publisher.id == session_id,
+        ]
+
+        titles = self.get_with_joins(
+            db,
+            model=model,
+            columns=columns,
+            join_models=join_models,
+            join_conditions=join_conditions,
+            filter_conditions=filter_conditions,
+            multiple=True,
+        )
+
+        return titles

db/query/query_bot.py

@@ -0,0 +1,57 @@
+from db.models import Bot, Session_Publisher
+from db.models import Session as SessionModel
+from db.query.base_query import BaseQuery
+
+
+class BotQuery(BaseQuery):
+    def __init__(self, user):
+        super().__init__(user)
+
+    def add_bot(self, db, session_id, bot, metadata_id, type="bot_one"):
+        # Create the new session
+        if type == "bot_one":
+            new_bot_one = Session_Publisher(
+                id=session_id,
+                user_id=self.user.get("id"),
+                bot_name=bot,
+                metadata_id=metadata_id,
+            )
+
+        else:
+            new_bot_one = SessionModel(
+                id=session_id,
+                user_id=self.user.get("id"),
+                bot_id=bot,
+                metadata_id=metadata_id,
+            )
+
+        return self.add(db, new_bot_one)
+
+    def get_session_ids_bot(self, db, metadata_id, type="bot_one"):
+        model = Session_Publisher
+        columns = [
+            Session_Publisher.id,
+            Session_Publisher.bot_name,
+            Session_Publisher.updated_at,
+        ]
+        filter_conditions = [
+            Session_Publisher.user_id == self.user.get("id"),
+            Session_Publisher.metadata_id == metadata_id,
+        ]
+
+        sessions = self.get_columns(db, model=model, columns=columns, filter_conditions=filter_conditions, multiple=True)
+
+        return sessions
+
+    def update_session_bot(self, db, bot_id, name):
+        update_data = {"name": name}
+        self.update(db, Bot, bot_id, update_data)
+
+    def delete_session_bot(self, db, bot_id):
+        self.delete(db, Bot, bot_id)
+
+    def get_bot(self, db, bot_id):
+        return self.get(db, Bot, bot_id)
+
+    # def get_all_bots(self, db):
+    #     return self.get_all(db, Bot)

db/query/query_category.py

@@ -0,0 +1,41 @@
+from db.models import Category, Metadata
+from db.query.base_query import BaseQuery
+from sqlalchemy import select, delete, update
+
+class CategoryQuery(BaseQuery):
+    def __init__(self, user):
+        super().__init__(user)
+
+    def add_category(self, db, name):
+        new_category = Category(name=name, user_id=self.user["id"])
+        return self.add(db, new_category)
+
+    def update_category(self, db, category_id, name):
+        update_data = {"name": name}
+        self.update(db, Category, category_id, update_data)
+
+    def delete_category(self, db, category_id):
+        self.delete(db, Category, category_id)
+
+    def get_category(self, db, category_id):
+        columns = [Category.category]
+        model = Category
+        return self.get_columns(db,model=model, columns=columns, id=category_id)
+
+    def get_current_category(self, db, metadata_id):
+        columns = [Category.category]
+        join_models = [Metadata]
+        where_conditions = [Metadata.id == metadata_id]
+
+        result = self.get_with_joins(
+            db,
+            join_models=join_models,
+            filter_conditions=where_conditions,
+            columns=columns,
+            multiple=False  # Assuming you want a single result
+        )
+        
+        return result
+
+    def get_all_categories(self, db):
+        return self.get(db, Category, multiple=True)

db/query/query_user_meta.py

@@ -0,0 +1,106 @@
+from sqlalchemy import select
+from db.models import User_Meta, Metadata, Category
+from db.query.base_query import BaseQuery
+
+
+class UserMetaQuery(BaseQuery):
+    def __init__(self, user):
+        super().__init__(user)
+
+    # def get_user_meta_entries(self, db):
+    #     """Fetch all user meta entries joined with metadata and category."""
+    #     join_models = [Metadata, Category]
+    #     print(join_models)
+    #     join_conditions = [
+    #         User_Meta.metadata_id == Metadata.id,
+    #         Metadata.category_id == Category.id,
+    #     ]
+    #     print(join_conditions)
+
+    #     result = self.get_all_with_joins(
+    #         db,
+    #         model=User_Meta,
+    #         join_models=join_models,
+    #         join_conditions=join_conditions,
+    #     )
+    #     return result
+    
+    def get_user_meta_entries(self, db):
+        """Fetch all user meta entries joined with metadata and category."""
+        join_models = [Metadata, Category]
+        join_conditions = [
+            User_Meta.metadata_id == Metadata.id,
+            Metadata.category_id == Category.id,
+        ]
+        
+        filter_conditions = [User_Meta.user_id == self.user_id]
+
+        result = self.get_with_joins(
+            db,
+            model=User_Meta,
+            join_models=join_models,
+            join_conditions=join_conditions,
+            filter_conditions=filter_conditions,
+            multiple=True
+        )
+        return result
+
+
+    def insert_user_meta_entries(self, db, metadata_ids):
+        """Insert new user meta entries."""
+        user_meta_entries = [
+            User_Meta(user_id=self.user.get("id"), metadata_id=mid)
+            for mid in metadata_ids
+        ]
+
+        # Use the method from BaseQuery to insert entries
+        self.insert_entries(db, user_meta_entries)
+        return {
+            "message": "User meta entries added successfully.",
+            "metadata_ids": metadata_ids,  # Include the metadata IDs in the result
+        }
+    
+    def update_user_meta_entries(self, db, metadata_ids):
+        """Update user meta entries: keep, delete, or add new entries based on metadata_ids."""
+        filter_conditions = [User_Meta.user_id == self.user_id]
+        
+        # Fetch existing user meta entries
+        existing_user_meta = self.get(db, model=User_Meta, filter_conditions=filter_conditions, multiple=True)
+        existing_user_meta = [user_meta[0] for user_meta in existing_user_meta]
+        existing_meta_ids = [entry.metadata_id for entry in existing_user_meta]
+
+        # Convert both lists to sets once for efficiency
+        metadata_ids_set = set(metadata_ids)
+        existing_meta_ids_set = set(existing_meta_ids)
+
+        # Find metadata to add, keep, or delete
+        metadata_to_add = list(metadata_ids_set - existing_meta_ids_set)
+        metadata_to_keep = list(metadata_ids_set & existing_meta_ids_set)
+        metadata_to_delete = list(existing_meta_ids_set - metadata_ids_set)
+
+        # Delete entries that are no longer in the updated metadata_ids list
+        if metadata_to_delete:
+            db.query(User_Meta).filter(User_Meta.user_id == self.user_id, User_Meta.metadata_id.in_(metadata_to_delete)).delete(synchronize_session=False)
+
+        # Add new entries for metadata that are not in the existing user meta
+        for meta_id in metadata_to_add:
+            new_entry = User_Meta(user_id=self.user_id, metadata_id=meta_id)
+            self.add(db, new_entry)
+
+        db.commit()
+        
+        return {
+            "status": "success",
+            "added_meta": list(metadata_to_add),
+            "deleted_meta": list(metadata_to_delete),
+            "kept_meta": list(metadata_to_keep),
+        }
+
+    def delete_user_meta(self, db, metadata_id):
+        """Delete user meta entries by metadata_id."""
+        self.delete(db, model=User_Meta, id=metadata_id)
+        return {"message": f"Book user with id {id} deleted successfully."}
+
+    def delete_all_user_meta(self, db):
+        """Delete all user meta entries for a user."""
+        self.delete_all(db, model=User_Meta)

script/document_uploader.py

@@ -8,6 +8,7 @@ from fastapi import UploadFile,status
 from fastapi.responses import JSONResponse
 
 from llama_index.core.node_parser import (
+    SentenceSplitter,
     SemanticSplitterNodeParser,
 )
 
@@ -82,16 +83,23 @@ class Uploader:
         #  Run the pipeline
         try:
             nodes_with_metadata = pipeline.run(documents=documents_with_metadata)
-            
+            # nodes_with_metadata = splitter.get_nodes_from_documents(documents_with_metadata)
             return nodes_with_metadata
         
         except Exception as e:
-            # Log the error and return JSONResponse for FastAPI
-            logging.error(f"An error occurred in making pipeline: {e}")
-            return JSONResponse(
-                status_code=500,
-                content="An internal server error occurred making pipeline.",
-            )
+            try:
+                # If the first method fails, fallback to sentence splitter
+                sentence_splitter = SentenceSplitter(chunk_size=512)
+                nodes_with_metadata = sentence_splitter.get_nodes_from_documents(documents_with_metadata)
+                print("Pipeline processing completed with SentenceSplitter fallback.")
+                return nodes_with_metadata
+            except Exception as fallback_error:
+                # Log the second error and return JSONResponse for FastAPI
+                logging.error(f"Error with SentenceSplitter fallback: {fallback_error}")
+                return JSONResponse(
+                    status_code=500,
+                    content="An internal server error occurred during pipeline processing.",
+                )
             
     def filter_document(self, documents):
         api_key = PINECONE_CONFIG.PINECONE_API_KEY

script/vector_db.py

@@ -98,10 +98,10 @@ class IndexManager:
 
         return ids
     
-    def delete_vector_database(self, old_reference):
+    def delete_vector_database(self, title):
         try :
             batch_size = 1000
-            all_ids = self.get_all_ids_from_index(old_reference['title'])
+            all_ids = self.get_all_ids_from_index(title)
             all_ids = list(all_ids)
             
             # Split ids into chunks of batch_size
@@ -115,11 +115,11 @@ class IndexManager:
         except Exception as e:
             return JSONResponse(status_code=500, content="An error occurred while delete metadata")    
 
-    def update_vector_database(self, old_reference, new_reference):
+    def update_vector_database(self, current_reference, new_reference):
         
         reference = new_reference
         
-        all_ids = self.get_all_ids_from_index(old_reference['title'])
+        all_ids = self.get_all_ids_from_index(current_reference['title'])
         all_ids = list(all_ids)
         
         for id in all_ids:

service/dto.py

@@ -96,7 +96,7 @@ class BotMetaCreate(BaseModel):
 
 
 class BotCreateRequest(BaseModel):
-    bot_name: str
+    name: str
 
 
 class BotResponse(BaseModel):

utils/error_handlers.py

@@ -0,0 +1,20 @@
+from fastapi.responses import JSONResponse
+
+
+def handle_exception(e: Exception):
+    """Helper function to handle exceptions in a consistent way."""
+    return JSONResponse(
+        status_code=500, content=f"An unexpected error occurred: {str(e)}"
+    )
+
+
+def handle_error(e, message):
+    return JSONResponse(status_code=500, content={"error": f"{message}: {str(e)}"})
+
+
+def not_found_error(message):
+    return JSONResponse(status_code=404, content={"error": message})
+
+
+def no_entries_found(message):
+    return JSONResponse(status_code=404, content={"message": message})