fix : change parsing

api/function.py

@@ -29,12 +29,12 @@ async def data_ingestion(reference, file: UploadFile) -> Any:
             user_id="admin_book_uploaded",
         )
         
-        # Upload to AWS
-        file_name = f"{reference['title']}"
-        aws_loader = Loader()
+        # # Upload to AWS
+        # file_name = f"{reference['title']}"
+        # aws_loader = Loader()
 
-        file_obj = file
-        aws_loader.upload_to_s3(file_obj, file_name)
+        # file_obj = file
+        # aws_loader.upload_to_s3(file_obj, file_name)
 
         uploader = Uploader(reference, file)
 

api/router/book.py

@@ -64,10 +64,6 @@ async def upload_file(
     if auth_response:
         return auth_response
 
-    # Create a new Metadata object
-    book_query = BookQuery(user)
-    book_query.add_book(db, title, author, category_id, year, publisher)
-
     logging.info("Database Inserted")
 
     # Query the category based on category_id
@@ -85,6 +81,10 @@ async def upload_file(
 
         # Process the file and handle data ingestion
         response = await data_ingestion(reference, file)
+        
+        # Create a new Metadata object
+        # book_query = BookQuery(user)
+        # book_query.add_book(db, title, author, category_id, year, publisher)
 
         return {
             "filename": file.filename,

api/router/category.py

@@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends
 
 from db.models import Category
 from db.database import get_db
+from db.query.query_category import CategoryQuery
 from service.dto import CategoryCreate
 
 from script.vector_db import IndexManager
@@ -19,13 +20,19 @@ db_dependency = Annotated[Session, Depends(get_db)]
 
 
 @router.get("/category")
-async def get_all_categories(user: user_dependency, db: db_dependency):
+async def get_all_categories_router(user: user_dependency, db: db_dependency):
     if user is None or user.get("role_id") != 1:
         return JSONResponse(status_code=401, content="Authentication Failed")
 
     try:
         # Logic to retrieve all categories
-        categories = db.query(Category).all()
+        category_query = CategoryQuery(user)
+        categories = category_query.get_all_categories(db)
+        
+        print(categories)
+        
+        
+        # categories = db.query(Category).all()
         if not categories:
             return JSONResponse(status_code=404, content="No categories found")
 
@@ -43,7 +50,7 @@ async def get_all_categories(user: user_dependency, db: db_dependency):
 
 
 @router.get("/category/{category_id}")
-async def get_categories_by_ids(
+async def get_category_by_id_router(
     user: user_dependency,
     db: db_dependency,
     category_id: int,
@@ -53,7 +60,8 @@ async def get_categories_by_ids(
 
     try:
         # Fetch categories based on the list of provided category_ids
-        category = db.query(Category).filter(Category.id == category_id).first()
+        category_query = CategoryQuery(user)
+        category = category_query.get_category_by_id(db, category_id)
 
         if category is None:
             return JSONResponse(status_code=404, content="No categories found for the given IDs")
@@ -75,21 +83,17 @@ async def create_category(user: user_dependency, db: db_dependency, category: Ca
 
     try:
         # Check if category already exists
-        existing_category = (
-            db.query(Category).filter(Category.category == category.category_name).first()
-        )
-        if existing_category:
+        category_query = CategoryQuery(user)
+        existing_category = category_query.get_existing_category(db, category.category_name)
+        if not isinstance(existing_category,JSONResponse):
             return JSONResponse(status_code=400, content="Category already exists")
-
-        # Logic to create a new category
-        new_category = Category(category=category)  # Assuming Category is your model
-        db.add(new_category)
-        db.commit()
-        db.refresh(new_category)
+        
+        # Add category
+        category_query.add_category(db, category.category_name)
+        print("category added")
 
         return {
             "message": "Category created successfully",
-            "category_id": new_category.id,
         }
 
     except IntegrityError:
@@ -99,13 +103,6 @@ async def create_category(user: user_dependency, db: db_dependency, category: Ca
             content="Database integrity error: possibly a duplicate entry.",
         )
 
-    except SQLAlchemyError as e:
-        db.rollback()
-        return JSONResponse(
-            status_code=500, content="Database error occurred: " + str(e)
-        )
-
-
 @router.put("/category/{category_id}")
 async def update_category(
     user: user_dependency, db: db_dependency, category_id: int, category: CategoryCreate

api/router/testing.py

@@ -0,0 +1,82 @@
+from fastapi import FastAPI, HTTPException, Depends, Form
+from fastapi.security import OAuth2PasswordBearer
+import httpx
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+app = FastAPI()
+
+# Bearer token for API authentication
+BEARER_TOKEN = os.getenv("MEDUCINE_API_BEARER_TOKEN")
+
+# Base URL for the Meducine API
+BASE_URL = os.getenv("BASE_URL")
+
+# OAuth2PasswordBearer provides the token as a dependency
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/login")
+
+@app.post("/login")
+async def login(email: str = Form(...), password: str = Form(...)):
+    async with httpx.AsyncClient() as client:
+        try:
+            response = await client.post(
+                f"{BASE_URL}/actions/meducine-restapi/auth/login",
+                data={"email": email, "password": password},
+                headers={"Authorization": f"Bearer {BEARER_TOKEN}"}
+            )
+            response.raise_for_status()  # Raise an error for bad responses (4xx or 5xx)
+            return handle_response(response)  # Assuming this function formats the response correctly
+        except httpx.HTTPStatusError as e:
+            raise HTTPException(status_code=e.response.status_code, detail=e.response.text)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/actions/meducine-restapi/auth/logout")
+async def logout(email: str = Form(...), password: str = Form(...)):
+    async with httpx.AsyncClient() as client:
+        response = await client.post(
+            f"{BASE_URL}/actions/meducine-restapi/auth/logout",
+            data={"email": email, "password": password},
+            headers={"Authorization": f"Bearer {BEARER_TOKEN}"}
+        )
+    return handle_response(response)
+
+@app.get("/actions/meducine-restapi/auth/identity")
+async def get_identity(token: str = Depends(oauth2_scheme)):
+    async with httpx.AsyncClient() as client:
+        response = await client.get(
+            f"{BASE_URL}/actions/meducine-restapi/auth/identity",
+            headers={"Authorization": f"Bearer {token}"}
+        )
+    return handle_response(response)
+
+@app.get("/actions/meducine-restapi/user/has-premium-access")
+async def check_premium_access(feature: str, token: str = Depends(oauth2_scheme)):
+    async with httpx.AsyncClient() as client:
+        response = await client.get(
+            f"{BASE_URL}/actions/meducine-restapi/user/has-premium-access",
+            params={"feature": feature},
+            headers={"Authorization": f"Bearer {token}"}
+        )
+    return handle_response(response)
+
+def handle_response(response: httpx.Response):
+    """
+    Handles the response from the Meducine API, returning appropriate responses based on status codes.
+    """
+    if response.status_code in range(200, 300):
+        return response.json()  # Successful request
+    elif response.status_code in range(400, 500):
+        raise HTTPException(status_code=response.status_code, detail=response.json())  # Client error
+    elif response.status_code in range(500, 600):
+        raise HTTPException(status_code=response.status_code, detail="Server error")  # Server error
+    else:
+        raise HTTPException(status_code=500, detail="Unexpected error")
+
+# Run the application
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="127.0.0.1", port=8000)

core/chat/bot_service.py

@@ -10,7 +10,7 @@ from llama_index.core.llms import MessageRole
 
 from core.chat.engine import Engine
 from core.chat.chatstore import ChatStore
-from core.parser import clean_text, update_response, renumber_sources
+from core.parser import clean_text, update_response, sort_and_renumber_sources
 
 from service.dto import ChatMessage
 from pymongo.mongo_client import MongoClient
@@ -47,7 +47,7 @@ class ChatCompletionService:
 
             # Update response and renumber sources
             response = update_response(str(response))
-            contents = renumber_sources(contents)
+            contents = sort_and_renumber_sources(contents)
 
             # Add contents to metadata
             metadata_collection = self._attach_contents_to_metadata(contents, metadata_collection)

core/parser.py

@@ -75,6 +75,29 @@ def renumber_sources(source_list):
     return new_sources
 
 
+def sort_and_renumber_sources(source_list):
+    """
+    This function takes a list of sources, sorts them based on the source number, 
+    and renumbers them sequentially starting from 1.
+    
+    :param source_list: List of strings containing source information.
+    :return: Sorted and renumbered list of sources.
+    """
+    
+    # Function to extract source number
+    def extract_source_number(source):
+        match = re.search(r"Source (\d+)", source)
+        return int(match.group(1)) if match else float('inf')
+    
+    # Sort sources based on the source number
+    sorted_sources = sorted(source_list, key=extract_source_number)
+    
+    # Reassign the numbering in the sorted sources
+    for idx, source in enumerate(sorted_sources, 1):
+        sorted_sources[idx-1] = re.sub(r"Source \d+", f"Source {idx}", source)
+    
+    return sorted_sources
+
 def seperate_to_list(text):
     # Step 1: Split the text by line breaks (\n)
     lines = text.split("\n")

db/query/base_query.py

@@ -81,21 +81,30 @@ class BaseQuery:
 
     def update(self, db, model, id, update_data, filter_conditions=None):
         """Update an entry by ID."""
+        # Define the initial query to fetch the entry
         query = select(model).where(model.id == id)
+
+        # Append additional filter conditions if provided
         if filter_conditions:
-            query = query.where(*filter_conditions)
+            query = query.where(model.id == id, *filter_conditions)
             
+        # Attempt to fetch the entry
         not_found_message = f"Entry with ID {id} not found."    
         entry = self._fetch(db, query, not_found_message, multiple=False)
         
+        # Check if the entry was found
         if isinstance(entry, JSONResponse):
             return entry
 
-        db.execute(
-            update(model)
-            .where(model.id == id, model.user_id == self.user_id)
-            .values(update_data)
-        )
+        # Prepare the update statement
+        stmt = update(model).where(model.id == id).values(update_data)
+        db.execute(stmt)
+
+        # If filter conditions were provided, apply them to the update as well
+        if filter_conditions:
+            filter_stmt = update(model).where(model.id == id, *filter_conditions).values(update_data)
+            db.execute(filter_stmt)
+
         return self._handle_commit(db)
 
     def update_entries(self, db, model, update_data, filter_conditions=None):
@@ -128,8 +137,6 @@ class BaseQuery:
         else:
             query = select(model)
 
-        # Apply filtering by user ID and optional conditions
-        query = query
         if id:
             query = query.where(model.id == id)
         if filter_conditions:

db/query/query_book.py

@@ -81,7 +81,12 @@ class BookQuery(BaseQuery):
 
     def update_book(self, db, book_id, title, author):
         update_data = {"title": title, "author": author}
-        self.update(db, Metadata, book_id, update_data)
+        self.update(
+            db,
+            model=Metadata,
+            id=book_id,
+            update_data=update_data,
+        )
 
     def delete_book(self, db, book_id):
         self.delete(db, Metadata, book_id)
@@ -92,15 +97,6 @@ class BookQuery(BaseQuery):
     def get_metadata_books(self, db, metadata_id):
         return self.get(db, Metadata, id=metadata_id)
 
-    # def get_title_from_session(self, db, metadata_id, session_id):
-    #     model = Session_Publisher
-    #     columns = [Metadata.title]
-    #     join_models = [Session_Publisher.id == session_id, Metadata.id == metadata_id]
-
-    #     titles = self.get_all_with_join_columns(db, model, columns, join_models)
-
-    #     return titles
-
     def get_title_from_session(self, db, metadata_id, session_id):
         model = Session_Publisher
         columns = [Metadata.title]

db/query/query_bot.py

@@ -39,7 +39,13 @@ class BotQuery(BaseQuery):
             Session_Publisher.metadata_id == metadata_id,
         ]
 
-        sessions = self.get_columns(db, model=model, columns=columns, filter_conditions=filter_conditions, multiple=True)
+        sessions = self.get_columns(
+            db,
+            model=model,
+            columns=columns,
+            filter_conditions=filter_conditions,
+            multiple=True,
+        )
 
         return sessions
 

db/query/query_category.py

@@ -1,17 +1,16 @@
 from db.models import Category, Metadata
 from db.query.base_query import BaseQuery
-from sqlalchemy import select, delete, update
 
 class CategoryQuery(BaseQuery):
     def __init__(self, user):
         super().__init__(user)
 
-    def add_category(self, db, name):
-        new_category = Category(name=name, user_id=self.user["id"])
+    def add_category(self, db, category_name):
+        new_category = Category(category=category_name)
         return self.add(db, new_category)
 
-    def update_category(self, db, category_id, name):
-        update_data = {"name": name}
+    def update_category(self, db, category_id, category_name):
+        update_data = {"category": category_name}
         self.update(db, Category, category_id, update_data)
 
     def delete_category(self, db, category_id):
@@ -20,7 +19,22 @@ class CategoryQuery(BaseQuery):
     def get_category(self, db, category_id):
         columns = [Category.category]
         model = Category
-        return self.get_columns(db,model=model, columns=columns, id=category_id)
+        results = self.get_columns(db,model=model, columns=columns, id=category_id)
+        return results
+    
+    def get_existing_category(self, db, category_name):
+        columns = [Category.category]
+        model = Category
+        filter_condition = [Category.category == category_name]
+        results = self.get_columns(db, columns=columns, model=model, filter_conditions=filter_condition)
+        return results
+    
+    def get_category_by_id(self, db, category_id):
+        model = Category
+        filter_conditions = [Category.id == category_id]
+        results = self.get(db, model=model, filter_conditions=filter_conditions)
+        
+        return results
 
     def get_current_category(self, db, metadata_id):
         columns = [Category.category]
@@ -38,4 +52,6 @@ class CategoryQuery(BaseQuery):
         return result
 
     def get_all_categories(self, db):
-        return self.get(db, Category, multiple=True)
+        results = self.get(db, Category, multiple=True)
+        categories = [category[0] for category in results]
+        return categories

db/query/query_user_meta.py

@@ -6,24 +6,6 @@ from db.query.base_query import BaseQuery
 class UserMetaQuery(BaseQuery):
     def __init__(self, user):
         super().__init__(user)
-
-    # def get_user_meta_entries(self, db):
-    #     """Fetch all user meta entries joined with metadata and category."""
-    #     join_models = [Metadata, Category]
-    #     print(join_models)
-    #     join_conditions = [
-    #         User_Meta.metadata_id == Metadata.id,
-    #         Metadata.category_id == Category.id,
-    #     ]
-    #     print(join_conditions)
-
-    #     result = self.get_all_with_joins(
-    #         db,
-    #         model=User_Meta,
-    #         join_models=join_models,
-    #         join_conditions=join_conditions,
-    #     )
-    #     return result
     
     def get_user_meta_entries(self, db):
         """Fetch all user meta entries joined with metadata and category."""

db/repository.py

@@ -1,36 +0,0 @@
-from databases import Database
-import datetime
-
-
-def get_db_conn(config):
-    db_url = f"{config.DB_URI}"
-    return Database(db_url)
-
-
-class Repository:
-    def __init__(self, db_conn):
-        self.db_conn = db_conn
-
-    async def get_by_query(self, query, param):
-        results = await self.db_conn.fetch_all(query, param)
-        print("result get _by query", results)
-        return [dict(result) for result in results]
-
-    async def _fetch_one(self, query, param):
-        result = await self.db_conn.fetch_one(query, param)
-        return dict(result) if result is not None else result
-
-    async def _exec(self, query, param):
-        return await self.db_conn.execute(query, param)
-
-    async def _exec_many(self, query, params):
-        return await self.db_conn.execute_many(query, params)
-
-    def update_params(self, params, update=False):
-        current_time = datetime.datetime.now()
-        if update == False:
-
-            params.update({"createdAt": current_time, "updatedAt": current_time})
-        else:
-            params.update({"updatedAt": current_time})
-        return params

script/document_uploader.py

@@ -2,15 +2,16 @@ from llama_index.core.ingestion import IngestionPipeline
 from llama_index.embeddings.openai import OpenAIEmbedding
 from config import PINECONE_CONFIG
 from pinecone.grpc import PineconeGRPC as Pinecone
-from service.reader import Reader
+# from service.reader import Reader
 from script.get_metadata import Metadata
-from fastapi import UploadFile,status
+from fastapi import UploadFile, status
 from fastapi.responses import JSONResponse
 
 from llama_index.core.node_parser import (
     SentenceSplitter,
     SemanticSplitterNodeParser,
 )
+from service.reader_v3 import upload_file
 
 # from script.get_topic import extract_topic
 
@@ -23,18 +24,18 @@ class Uploader:
     def __init__(self, reference, file: UploadFile):
         self.file = file
         # self.content_table = content_table
-        self.reader = Reader()
+        # self.reader = Reader()
         self.reference = reference
         self.metadata = Metadata(reference)
 
-    async def ingest_documents(self, file: UploadFile):
-        """Load documents from the storage path."""
-        documents = await self.reader.read_from_uploadfile(file)
-        print("Banyak document : ", len(documents))
-        print("document successfully ingested")
+    # async def ingest_documents(self, file: UploadFile):
+    #     """Load documents from the storage path."""
+    #     documents = await self.reader.read_from_uploadfile(file)
+    #     print("Banyak document : ", len(documents))
+    #     print("document successfully ingested")
+
+    #     return documents
 
-        return documents
-    
     def check_existing_metadata(self, pinecone_index, title, random_vector):
         try:
             result = pinecone_index.query(
@@ -53,17 +54,17 @@ class Uploader:
 
     async def process_documents(self):
         # Ingest documents
-        documents = await self.ingest_documents(self.file)
-
-        # topic_extractor = extract_topic(self.reference, self.content_table)
-
-        embed_model = OpenAIEmbedding()
+        # documents = await self.ingest_documents(self.file)
 
         # Get metadata
-        documents_with_metadata = self.metadata.apply_metadata(documents)
-        
+        # documents_with_metadata = self.metadata.apply_metadata(documents)
+        documents_with_metadata = await upload_file(self.reference, self.file)
+
+        # Get Topic
+        # topic_extractor = extract_topic(self.reference, self.content_table)
         # document_filtered = self.filter_document(documents_with_metadata)
 
+        embed_model = OpenAIEmbedding()
         # Set up the ingestion pipeline
         pipeline = IngestionPipeline(
             transformations=[
@@ -75,7 +76,7 @@ class Uploader:
                 # topic_extractor,
             ]
         )
-        
+
         # splitter = SemanticSplitterNodeParser(
         #     buffer_size=1, breakpoint_percentile_threshold=95, embed_model=embed_model
         # )
@@ -85,12 +86,14 @@ class Uploader:
             nodes_with_metadata = pipeline.run(documents=documents_with_metadata)
             # nodes_with_metadata = splitter.get_nodes_from_documents(documents_with_metadata)
             return nodes_with_metadata
-        
+
         except Exception as e:
             try:
                 # If the first method fails, fallback to sentence splitter
                 sentence_splitter = SentenceSplitter(chunk_size=512)
-                nodes_with_metadata = sentence_splitter.get_nodes_from_documents(documents_with_metadata)
+                nodes_with_metadata = sentence_splitter.get_nodes_from_documents(
+                    documents_with_metadata
+                )
                 print("Pipeline processing completed with SentenceSplitter fallback.")
                 return nodes_with_metadata
             except Exception as fallback_error:
@@ -100,7 +103,7 @@ class Uploader:
                     status_code=500,
                     content="An internal server error occurred during pipeline processing.",
                 )
-            
+
     def filter_document(self, documents):
         api_key = PINECONE_CONFIG.PINECONE_API_KEY
         client = Pinecone(api_key=api_key)
@@ -117,4 +120,4 @@ class Uploader:
             if len(result) == 0:
                 filtered_documents.append(doc)
 
-        return filtered_documents
+        return filtered_documents

service/reader_v3.py

@@ -0,0 +1,86 @@
+import os
+import nest_asyncio
+
+from llama_parse import LlamaParse
+from llama_index.core.node_parser import SimpleNodeParser
+from dotenv import load_dotenv
+from fastapi import UploadFile, File
+from fastapi.responses import JSONResponse
+import fitz
+
+from script.get_metadata import Metadata
+
+load_dotenv()
+nest_asyncio.apply()
+
+
+def parse_journal(content: bytes, file_name: str):
+    """Parse the journal using LlamaParse."""
+    try:
+        # Initialize the parser
+        parser = LlamaParse(
+            api_key=os.getenv("LLAMA_PARSE_API_KEY"),
+            result_type="markdown",
+            use_vendor_multimodal_model=True,
+            vendor_multimodal_model_name="openai-gpt-4o-mini",
+        )
+
+        # Load and process the document
+        llama_parse_documents = parser.load_data(
+            content, extra_info={"file_name": file_name}
+        )
+
+        return llama_parse_documents
+
+    except Exception as e:
+        return JSONResponse(status_code=400, content=f"Error processing file: {e}")
+
+
+async def extract_metadata(content: bytes):
+    """Extract metadata from the PDF content."""
+    try:
+        # Open the binary content with PyMuPDF
+        pdf_document = fitz.open("pdf", content)  # "pdf" specifies the format
+
+        # Extract metadata
+        metadata = pdf_document.metadata
+
+        # Prepare metadata dictionary with default values for missing fields
+        metadata_dict = {
+            "title": metadata.get("title", "N/A"),
+            "author": metadata.get("author", "N/A"),
+            "subject": metadata.get("subject", "N/A"),
+            "keywords": metadata.get("keywords", "N/A"),
+            "creation_date": metadata.get("created", "N/A"),
+            "modification_date": metadata.get("modified", "N/A"),
+        }
+
+        return metadata_dict
+
+    except Exception as e:
+        return JSONResponse(status_code=500, content=f"Error inputting metadata: {e}")
+
+
+async def upload_file(reference, file: UploadFile):
+    try:
+        # Read the binary content of the uploaded file once
+        content = await file.read()
+        # Parse the journal
+        parsed_documents = parse_journal(content, file.filename)
+        # Extract metadata
+        # metadata_dict = await extract_metadata(content)
+        # print("Metadata Dictionary : \n\n", metadata_dict)
+
+        metadata_gen = Metadata(reference)
+        documents_with_metadata = metadata_gen.apply_metadata(parsed_documents)
+        
+        # document_with_metadata = 
+
+        print("Document with Metadata : \n\n", documents_with_metadata)
+        print("Banyak documents : \n", len(documents_with_metadata))
+
+        # Return both parsed documents and metadata
+        return documents_with_metadata
+
+    except Exception as e:
+        return JSONResponse(status_code=500, content=f"Error processing file: {e}")

utils/error_handlers.py

@@ -9,12 +9,12 @@ def handle_exception(e: Exception):
 
 
 def handle_error(e, message):
-    return JSONResponse(status_code=500, content={"error": f"{message}: {str(e)}"})
+    return JSONResponse(status_code=500, content={"error occurs": f"{message}: {str(e)}"})
 
 
 def not_found_error(message):
-    return JSONResponse(status_code=404, content={"error": message})
+    return JSONResponse(status_code=404, content={"not found message": message})
 
 
 def no_entries_found(message):
-    return JSONResponse(status_code=404, content={"message": message})
+    return JSONResponse(status_code=404, content={"no entries found": message})