import os from fastapi import FastAPI, Depends, HTTPException from fastapi.responses import JSONResponse, RedirectResponse from fastapi.middleware.gzip import GZipMiddleware from pydantic import BaseModel from typing import List, Dict from datasets import Dataset from src.api.models.embedding_models import ( CreateEmbeddingRequest, ReadEmbeddingRequest, UpdateEmbeddingRequest, DeleteEmbeddingRequest, EmbedRequest, SearchEmbeddingRequest, ) from src.api.database import get_db, Database, QueryExecutionError, HealthCheckError from src.api.services.embedding_service import EmbeddingService from src.api.services.huggingface_service import HuggingFaceService from src.api.exceptions import DatasetNotFoundError, DatasetPushError, OpenAIError import logging from dotenv import load_dotenv # Load environment variables load_dotenv() # Set up structured logging logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) logger = logging.getLogger(__name__) description = """A FastAPI application for similarity search with PostgreSQL and OpenAI embeddings. Direct/API URL: https://re-mind-similarity-search.hf.space """ # Initialize FastAPI app app = FastAPI( title="Similarity Search API", description=description, version="1.0.0", ) app.add_middleware(GZipMiddleware, minimum_size=1000) # Dependency to get EmbeddingService def get_embedding_service() -> EmbeddingService: return EmbeddingService(openai_api_key=os.getenv("OPENAI_API_KEY")) # Dependency to get HuggingFaceService def get_huggingface_service() -> HuggingFaceService: return HuggingFaceService() # Root endpoint redirects to /docs @app.get("/") async def root(): return RedirectResponse(url="/docs") # Health check endpoint @app.get("/health") async def health_check(db: Database = Depends(get_db)): try: is_healthy = await db.health_check() if not is_healthy: raise HTTPException(status_code=500, detail="Database is unhealthy") return {"status": "healthy"} except HealthCheckError as e: raise HTTPException(status_code=500, detail=str(e)) # Endpoint to generate embeddings for a list of strings @app.post("/embed") async def embed( request: EmbedRequest, embedding_service: EmbeddingService = Depends(get_embedding_service), ): """ Generate embeddings for a list of strings and return them in the response. """ try: # Step 1: Generate embeddings logger.info("Generating embeddings for list of texts...") embeddings = await embedding_service.create_embeddings(request.texts) return JSONResponse( content={ "message": "Embeddings generated successfully.", "embeddings": embeddings, "num_texts": len(request.texts), } ) except OpenAIError as e: logger.error(f"OpenAI API error: {e}") raise HTTPException(status_code=500, detail=f"OpenAI API error: {e}") except Exception as e: logger.error(f"An error occurred: {e}") raise HTTPException(status_code=500, detail=f"An error occurred: {e}") # Endpoint to create embeddings from a database query @app.post("/create_embeddings") async def create_embedding( request: CreateEmbeddingRequest, db: Database = Depends(get_db), embedding_service: EmbeddingService = Depends(get_embedding_service), huggingface_service: HuggingFaceService = Depends(get_huggingface_service), ): """ Create embeddings for the target column in the dataset. """ try: embedding_service.model = request.model embedding_service.batch_size = request.batch_size # embedding_service.max_concurrent_requests = request.max_concurrent_requests # Step 1: Query the database logger.info("Fetching data from the database...") result = await db.fetch(request.query) # logger.info(f"{result}") dataset = Dataset.from_dict(result) # Step 2: Generate embeddings dataset = await embedding_service.create_embeddings( dataset, request.target_column, request.output_column ) # Step 3: Push to Hugging Face Hub await huggingface_service.push_to_hub(dataset, request.dataset_name) return JSONResponse( content={ "message": "Embeddings created and pushed to Hugging Face Hub.", "dataset_name": request.dataset_name, "num_rows": len(dataset), } ) except QueryExecutionError as e: logger.error(f"Database query failed: {e}") raise HTTPException(status_code=500, detail=f"Database query failed: {e}") except OpenAIError as e: logger.error(f"OpenAI API error: {e}") raise HTTPException(status_code=500, detail=f"OpenAI API error: {e}") except DatasetPushError as e: logger.error(f"Failed to push dataset: {e}") raise HTTPException(status_code=500, detail=f"Failed to push dataset: {e}") except Exception as e: logger.error(f"An error occurred: {e}") raise HTTPException(status_code=500, detail=f"An error occurred: {e}") # Endpoint to read embeddings @app.post("/read_embeddings") async def read_embeddings( request: ReadEmbeddingRequest, huggingface_service: HuggingFaceService = Depends(get_huggingface_service), ): """ Read embeddings from a Hugging Face dataset. """ try: dataset = await huggingface_service.read_dataset(request.dataset_name) return dataset.to_dict() except DatasetNotFoundError as e: logger.error(f"Dataset not found: {e}") raise HTTPException(status_code=404, detail=f"Dataset not found: {e}") except Exception as e: logger.error(f"An error occurred: {e}") raise HTTPException(status_code=500, detail=f"An error occurred: {e}") # Endpoint to update embeddings @app.post("/update_embeddings") async def update_embeddings( request: UpdateEmbeddingRequest, huggingface_service: HuggingFaceService = Depends(get_huggingface_service), ): """ Update embeddings in a Hugging Face dataset by generating embeddings for new data and concatenating it with the existing dataset. """ try: # Call the update_dataset method to generate embeddings, concatenate, and push the updated dataset updated_dataset = await huggingface_service.update_dataset( request.dataset_name, request.updates, request.target_column, request.output_column, ) return { "message": "Embeddings updated successfully.", "dataset_name": request.dataset_name, "num_rows": len(updated_dataset), } except DatasetPushError as e: logger.error(f"Failed to update dataset: {e}") raise HTTPException(status_code=500, detail=f"Failed to update dataset: {e}") except Exception as e: logger.error(f"An error occurred: {e}") raise HTTPException(status_code=500, detail=f"An error occurred: {e}") # Endpoint to delete embeddings @app.post("/delete_embeddings") async def delete_embeddings( request: DeleteEmbeddingRequest, huggingface_service: HuggingFaceService = Depends(get_huggingface_service), ): """ Delete embeddings from a Hugging Face dataset. """ try: await huggingface_service.delete_dataset(request.dataset_name) return { "message": "Embeddings deleted successfully.", "dataset_name": request.dataset_name, } except DatasetPushError as e: logger.error(f"Failed to delete columns: {e}") raise HTTPException(status_code=500, detail=f"Failed to delete columns: {e}") except Exception as e: logger.error(f"An error occurred: {e}") raise HTTPException(status_code=500, detail=f"An error occurred: {e}") @app.post("/search_embeddings") async def search_embedding( request: SearchEmbeddingRequest, embedding_service: EmbeddingService = Depends(get_embedding_service), huggingface_service: HuggingFaceService = Depends(get_huggingface_service), ): """ Search for similar texts in a dataset using embeddings. """ try: # Step 1: Generate embeddings for the query texts logger.info("Generating embeddings for query texts...") query_embeddings = await embedding_service.create_embeddings(request.texts) # Step 2: Load the dataset from Hugging Face Hub logger.info(f"Loading dataset from Hugging Face Hub: {request.dataset_name}...") dataset = await huggingface_service.read_dataset(request.dataset_name) # Step 3: Perform cosine similarity search logger.info("Performing cosine similarity search...") results = await embedding_service.search_embeddings( query_embeddings, dataset, request.embedding_column, request.target_column, request.num_results, request.additional_columns, ) return JSONResponse( content={ "message": "Search completed successfully.", "results": results, } ) except DatasetNotFoundError as e: logger.error(f"Dataset not found: {e}") raise HTTPException(status_code=404, detail=f"Dataset not found: {e}") except Exception as e: logger.error(f"An error occurred: {e}") raise HTTPException(status_code=500, detail=f"An error occurred: {e}")