Debug - Read Endpoint

src/api/database.py

@@ -1,384 +1,3 @@
-# import logging
-# from typing import Dict, List, Optional, AsyncGenerator
-# from pydantic import BaseSettings, PostgresDsn
-# import pg8000
-# from pg8000 import Connection, Cursor
-# from pg8000.exceptions import DatabaseError
-# import asyncio
-# from contextlib import asynccontextmanager
-# from dataclasses import dataclass
-# from threading import Lock
-
-# # Set up structured logging
-# logging.basicConfig(
-#     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-# )
-# logger = logging.getLogger(__name__)
-
-
-# class DatabaseSettings(BaseSettings):
-#     db_url: PostgresDsn
-#     pool_size: int = 5
-
-#     class Config:
-#         env_file = ".env"
-
-
-# @dataclass
-# class DatabaseConfig:
-#     username: str
-#     password: str
-#     hostname: str
-#     port: int
-#     database: str
-
-
-# class DatabaseError(Exception):
-#     """Custom exception for database errors."""
-
-#     pass
-
-
-# class Database:
-#     def __init__(self, db_url: str, pool_size: int):
-#         self.db_url = db_url
-#         self.pool_size = pool_size
-#         self.pool: List[Connection] = []
-#         self.lock = Lock()
-#         self.config = self._parse_db_url()
-
-#     def _parse_db_url(self) -> DatabaseConfig:
-#         """Parse the database URL into components."""
-#         result = urlparse(self.db_url)
-#         return DatabaseConfig(
-#             username=result.username,
-#             password=result.password,
-#             hostname=result.hostname,
-#             port=result.port or 5432,
-#             database=result.path.lstrip("/"),
-#         )
-
-#     async def connect(self) -> None:
-#         """Create a connection pool."""
-#         try:
-#             for _ in range(self.pool_size):
-#                 conn = await self._create_connection()
-#                 self.pool.append(conn)
-#             logger.info(
-#                 f"Database connection pool created with {self.pool_size} connections."
-#             )
-#         except DatabaseError as e:
-#             logger.error(f"Failed to create database connection pool: {e}")
-#             raise
-
-#     async def _create_connection(self) -> Connection:
-#         """Create a single database connection."""
-#         try:
-#             conn = pg8000.connect(
-#                 user=self.config.username,
-#                 password=self.config.password,
-#                 host=self.config.hostname,
-#                 port=self.config.port,
-#                 database=self.config.database,
-#             )
-#             return conn
-#         except DatabaseError as e:
-#             logger.error(f"Failed to create database connection: {e}")
-#             raise DatabaseError("Failed to create database connection.")
-
-#     async def disconnect(self) -> None:
-#         """Close all connections in the pool."""
-#         with self.lock:
-#             for conn in self.pool:
-#                 conn.close()
-#             self.pool.clear()
-#             logger.info("Database connection pool closed.")
-
-#     @asynccontextmanager
-#     async def get_connection(self) -> AsyncGenerator[Connection, None]:
-#         """Acquire a connection from the pool."""
-#         with self.lock:
-#             if not self.pool:
-#                 raise DatabaseError("Database connection pool is empty.")
-#             conn = self.pool.pop()
-#         try:
-#             yield conn
-#         finally:
-#             with self.lock:
-#                 self.pool.append(conn)
-
-#     async def fetch(self, query: str, *args) -> List[Dict]:
-#         """
-#         Execute a SELECT query and return the results as a list of dictionaries.
-
-#         Args:
-#             query (str): The SQL query to execute.
-#             *args: Query parameters.
-
-#         Returns:
-#             List[Dict]: A list of dictionaries where keys are column names and values are column values.
-#         """
-#         try:
-#             async with self.get_connection() as conn:
-#                 cursor: Cursor = conn.cursor()
-#                 cursor.execute(query, args)
-#                 rows = cursor.fetchall()
-#                 columns = [desc[0] for desc in cursor.description]
-#                 return [dict(zip(columns, row)) for row in rows]
-#         except DatabaseError as e:
-#             logger.error(f"Error executing query: {query}. Error: {e}")
-#             raise DatabaseError(f"Failed to execute query: {query}")
-
-#     async def execute(self, query: str, *args) -> None:
-#         """
-#         Execute an INSERT, UPDATE, or DELETE query.
-
-#         Args:
-#             query (str): The SQL query to execute.
-#             *args: Query parameters.
-#         """
-#         try:
-#             async with self.get_connection() as conn:
-#                 cursor: Cursor = conn.cursor()
-#                 cursor.execute(query, args)
-#                 conn.commit()
-#         except DatabaseError as e:
-#             logger.error(f"Error executing query: {query}. Error: {e}")
-#             raise DatabaseError(f"Failed to execute query: {query}")
-
-
-# # Dependency to get the database instance
-# async def get_db() -> AsyncGenerator[Database, None]:
-#     settings = DatabaseSettings()
-#     db = Database(db_url=settings.db_url, pool_size=settings.pool_size)
-#     await db.connect()
-#     try:
-#         yield db
-#     finally:
-#         await db.disconnect()
-
-
-# # Example usage
-# if __name__ == "__main__":
-
-#     async def main():
-#         settings = DatabaseSettings()
-#         db = Database(db_url=settings.db_url, pool_size=settings.pool_size)
-#         await db.connect()
-
-#         try:
-#             # Example query
-#             query = """
-# SELECT
-#     ppt.type AS product_type,
-#     pc.name AS product_category
-# FROM
-#     product_producttype ppt
-# INNER JOIN
-#     product_category pc
-# ON
-#     ppt.category_id = pc.id
-# """
-#             result = await db.fetch(query)
-#             print(result)
-#         finally:
-#             await db.disconnect()
-
-#     asyncio.run(main())
-
-# import logging
-# from urllib.parse import urlparse
-# from typing import Dict, List, Optional, AsyncGenerator
-# from pydantic_settings import BaseSettings
-# from pydantic import PostgresDsn
-# import pg8000
-# from pg8000 import Connection, Cursor
-# from pg8000.exceptions import DatabaseError
-# import asyncio
-# from contextlib import asynccontextmanager
-# from dataclasses import dataclass
-# from threading import Lock
-
-# # Set up structured logging
-# logging.basicConfig(
-#     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-# )
-# logger = logging.getLogger(__name__)
-
-
-# class DatabaseSettings(BaseSettings):
-#     db_url: PostgresDsn
-#     pool_size: int = 5
-
-#     class Config:
-#         env_file = ".env"
-
-
-# @dataclass
-# class DatabaseConfig:
-#     username: str
-#     password: str
-#     hostname: str
-#     port: int
-#     database: str
-
-
-# class DatabaseError(Exception):
-#     """Custom exception for database errors."""
-
-#     pass
-
-
-# class Database:
-#     def __init__(self, db_url: str, pool_size: int):
-#         self.db_url = db_url
-#         self.pool_size = pool_size
-#         self.pool: List[Connection] = []
-#         self.lock = Lock()
-#         self.config = self._parse_db_url()
-
-#     def _parse_db_url(self) -> DatabaseConfig:
-#         """Parse the database URL into components."""
-#         # Convert PostgresDsn to a string
-#         db_url_str = str(self.db_url)
-#         result = urlparse(db_url_str)
-#         return DatabaseConfig(
-#             username=result.username,
-#             password=result.password,
-#             hostname=result.hostname,
-#             port=result.port or 5432,
-#             database=result.path.lstrip("/"),
-#         )
-
-#     async def connect(self) -> None:
-#         """Create a connection pool."""
-#         try:
-#             for _ in range(self.pool_size):
-#                 conn = await self._create_connection()
-#                 self.pool.append(conn)
-#             logger.info(
-#                 f"Database connection pool created with {self.pool_size} connections."
-#             )
-#         except DatabaseError as e:
-#             logger.error(f"Failed to create database connection pool: {e}")
-#             raise
-
-#     async def _create_connection(self) -> Connection:
-#         """Create a single database connection."""
-#         try:
-#             conn = pg8000.connect(
-#                 user=self.config.username,
-#                 password=self.config.password,
-#                 host=self.config.hostname,
-#                 port=self.config.port,
-#                 database=self.config.database,
-#             )
-#             return conn
-#         except DatabaseError as e:
-#             logger.error(f"Failed to create database connection: {e}")
-#             raise DatabaseError("Failed to create database connection.")
-
-#     async def disconnect(self) -> None:
-#         """Close all connections in the pool."""
-#         with self.lock:
-#             for conn in self.pool:
-#                 conn.close()
-#             self.pool.clear()
-#             logger.info("Database connection pool closed.")
-
-#     @asynccontextmanager
-#     async def get_connection(self) -> AsyncGenerator[Connection, None]:
-#         """Acquire a connection from the pool."""
-#         with self.lock:
-#             if not self.pool:
-#                 raise DatabaseError("Database connection pool is empty.")
-#             conn = self.pool.pop()
-#         try:
-#             yield conn
-#         finally:
-#             with self.lock:
-#                 self.pool.append(conn)
-
-#     async def fetch(self, query: str, *args) -> List[Dict]:
-#         """
-#         Execute a SELECT query and return the results as a list of dictionaries.
-
-#         Args:
-#             query (str): The SQL query to execute.
-#             *args: Query parameters.
-
-#         Returns:
-#             List[Dict]: A list of dictionaries where keys are column names and values are column values.
-#         """
-#         try:
-#             async with self.get_connection() as conn:
-#                 cursor: Cursor = conn.cursor()
-#                 cursor.execute(query, args)
-#                 rows = cursor.fetchall()
-#                 columns = [desc[0] for desc in cursor.description]
-#                 return [dict(zip(columns, row)) for row in rows]
-#         except DatabaseError as e:
-#             logger.error(f"Error executing query: {query}. Error: {e}")
-#             raise DatabaseError(f"Failed to execute query: {query}")
-
-#     async def execute(self, query: str, *args) -> None:
-#         """
-#         Execute an INSERT, UPDATE, or DELETE query.
-
-#         Args:
-#             query (str): The SQL query to execute.
-#             *args: Query parameters.
-#         """
-#         try:
-#             async with self.get_connection() as conn:
-#                 cursor: Cursor = conn.cursor()
-#                 cursor.execute(query, args)
-#                 conn.commit()
-#         except DatabaseError as e:
-#             logger.error(f"Error executing query: {query}. Error: {e}")
-#             raise DatabaseError(f"Failed to execute query: {query}")
-
-
-# # Dependency to get the database instance
-# async def get_db() -> AsyncGenerator[Database, None]:
-#     settings = DatabaseSettings()
-#     db = Database(db_url=settings.db_url, pool_size=settings.pool_size)
-#     await db.connect()
-#     try:
-#         yield db
-#     finally:
-#         await db.disconnect()
-
-
-# # Example usage
-# if __name__ == "__main__":
-
-#     async def main():
-#         settings = DatabaseSettings()
-#         db = Database(db_url=settings.db_url, pool_size=settings.pool_size)
-#         await db.connect()
-
-#         try:
-#             # Example query
-#             query = "SELECT * FROM your_table LIMIT 10"
-#             query = """
-# SELECT
-#     ppt.type AS product_type,
-#     pc.name AS product_category
-# FROM
-#     product_producttype ppt
-# INNER JOIN
-#     product_category pc
-# ON
-#     ppt.category_id = pc.id
-# """
-#             result = await db.fetch(query)
-#             print(result)
-#         finally:
-#             await db.disconnect()
-
-#     asyncio.run(main())
-
 import logging
 from typing import AsyncGenerator, List, Optional, Dict
 from pydantic_settings import BaseSettings

src/api/models/embedding_models.py

@@ -13,6 +13,10 @@ class CreateEmbeddingRequest(BaseModel):
     dataset_name: str = "re-mind/product_type_embedding"
 
 
+class ReadEmbeddingRequest(BaseModel):
+    dataset_name: str
+
+
 class UpdateEmbeddingRequest(BaseModel):
     dataset_name: str
     updates: Dict[str, List]  # Column name -> List of values

src/api/services/huggingface_service.py

@@ -1,73 +1,3 @@
-# from datasets import Dataset, load_dataset
-# import logging
-# from typing import Optional, Dict, List
-# import pandas as pd
-# from src.api.exceptions import DatasetNotFoundError, DatasetPushError
-
-# # Set up structured logging
-# logging.basicConfig(
-#     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-# )
-# logger = logging.getLogger(__name__)
-
-
-# class HuggingFaceService:
-#     async def push_to_hub(self, df: pd.DataFrame, dataset_name: str) -> None:
-#         """Push the dataset to Hugging Face Hub."""
-#         try:
-#             logger.info(f"Creating Hugging Face Dataset: {dataset_name}...")
-#             ds = Dataset.from_pandas(df)
-#             ds.push_to_hub(dataset_name)
-#             logger.info(f"Dataset pushed to Hugging Face Hub: {dataset_name}")
-#         except Exception as e:
-#             logger.error(f"Failed to push dataset to Hugging Face Hub: {e}")
-#             raise DatasetPushError(f"Failed to push dataset: {e}")
-
-#     async def read_dataset(self, dataset_name: str) -> Optional[pd.DataFrame]:
-#         """Read a dataset from Hugging Face Hub."""
-#         try:
-#             logger.info(f"Loading dataset from Hugging Face Hub: {dataset_name}...")
-#             ds = load_dataset(dataset_name)
-#             df = ds["train"].to_pandas()
-#             return df
-#         except Exception as e:
-#             logger.error(f"Failed to read dataset: {e}")
-#             raise DatasetNotFoundError(f"Dataset not found: {e}")
-
-#     async def update_dataset(
-#         self, dataset_name: str, updates: Dict[str, List]
-#     ) -> Optional[pd.DataFrame]:
-#         """Update a dataset on Hugging Face Hub."""
-#         try:
-#             df = await self.read_dataset(dataset_name)
-#             for column, values in updates.items():
-#                 if column in df.columns:
-#                     df[column] = values
-#                 else:
-#                     logger.warning(f"Column '{column}' not found in dataset.")
-#             await self.push_to_hub(df, dataset_name)
-#             return df
-#         except Exception as e:
-#             logger.error(f"Failed to update dataset: {e}")
-#             raise DatasetPushError(f"Failed to update dataset: {e}")
-
-#     async def delete_columns(
-#         self, dataset_name: str, columns: List[str]
-#     ) -> Optional[pd.DataFrame]:
-#         """Delete columns from a dataset on Hugging Face Hub."""
-#         try:
-#             df = await self.read_dataset(dataset_name)
-#             for column in columns:
-#                 if column in df.columns:
-#                     df.drop(column, axis=1, inplace=True)
-#                 else:
-#                     logger.warning(f"Column '{column}' not found in dataset.")
-#             await self.push_to_hub(df, dataset_name)
-#             return df
-#         except Exception as e:
-#             logger.error(f"Failed to delete columns: {e}")
-#             raise DatasetPushError(f"Failed to delete columns: {e}")
-
 from datasets import Dataset, load_dataset
 from huggingface_hub import HfApi, HfFolder
 import logging

src/main.py

@@ -5,6 +5,7 @@ from pydantic import BaseModel
 from typing import List, Dict
 from src.api.models.embedding_models import (
     CreateEmbeddingRequest,
+    ReadEmbeddingRequest,
     UpdateEmbeddingRequest,
     DeleteEmbeddingRequest,
 )
@@ -114,16 +115,17 @@ async def create_embedding(
 
 
 # Endpoint to read embeddings
-@app.get("/read_embeddings/{dataset_name}")
+# @app.get("/read_embeddings/{dataset_name}")
+@app.get("/read_embeddings")
 async def read_embeddings(
-    dataset_name: str,
+    request: ReadEmbeddingRequest,
     huggingface_service: HuggingFaceService = Depends(get_huggingface_service),
 ):
     """
     Read embeddings from a Hugging Face dataset.
     """
     try:
-        df = await huggingface_service.read_dataset(dataset_name)
+        df = await huggingface_service.read_dataset(request.dataset_name)
         return df.to_dict(orient="records")
     except DatasetNotFoundError as e:
         logger.error(f"Dataset not found: {e}")
@@ -168,9 +170,7 @@ async def delete_embeddings(
     Delete embeddings from a Hugging Face dataset.
     """
     try:
-        await huggingface_service.delete_dataset(
-            request.dataset_name
-        )
+        await huggingface_service.delete_dataset(request.dataset_name)
         return {
             "message": "Embeddings deleted successfully.",
             "dataset_name": request.dataset_name,