test MarkitDown

app/main.py

@@ -10,11 +10,13 @@ from typing import Dict, List
 from prometheus_client import Counter, Histogram, start_http_server
 from pydantic import BaseModel, ValidationError
 from app.services.message import generate_reply, send_reply
-import logging
+import logging, httpx
 from datetime import datetime
 from sentence_transformers import SentenceTransformer
 from app.search.rag_pipeline import RAGSystem
 from contextlib import asynccontextmanager
+
+from httpx import ConnectTimeout, ReadTimeout, HTTPStatusError
 # from app.db.database import create_indexes, init_db
 from app.services.webhook_handler import verify_webhook
 from app.handlers.message_handler import MessageHandler
@@ -26,6 +28,7 @@ from app.api.api_prompt import prompt_router
 from app.api.api_file import file_router
 from app.utils.load_env import ACCESS_TOKEN, WHATSAPP_API_URL, GEMINI_API
 
+
 from markitdown import MarkItDown
 
 # Configure logging
@@ -152,17 +155,34 @@ async def webhook(request: Request):
 app.get("/webhook")(verify_webhook)
 
 @app.post("/load_file")
-async def load_file_with_markitdown(file_path:str, llm_client:str=None, model:str=None):
-    
-    if llm_client and model:
-        markitdown = MarkItDown(llm_client, model)
-        documents = markitdown.convert(file_path)
-    else:
-        markitdown = MarkItDown()
-        documents = markitdown.convert(file_path)
-
-    print(f"documents: {documents}")
-    return documents
+async def load_file_with_markitdown(file_path: str, llm_client: str = None, model: str = None):
+    try:
+        async with httpx.AsyncClient(timeout=10) as client:
+            response = await client.get(file_path)
+            response.raise_for_status()
+            content = response.text
+
+        # Initialize MarkItDown
+        if llm_client and model:
+            markitdown = MarkItDown(llm_client, model)
+        else:
+            markitdown = MarkItDown()
+
+        # Convert the fetched content
+        documents = markitdown.convert_content(content)  # Assuming a method that accepts raw content
+
+        print(f"documents: {documents}")
+        return {"documents": documents}
+
+    except ConnectTimeout:
+        raise HTTPException(status_code=504, detail="Connection to the external server timed out.")
+    except ReadTimeout:
+        raise HTTPException(status_code=504, detail="The external server took too long to respond.")
+    except HTTPStatusError as e:
+        raise HTTPException(status_code=e.response.status_code, detail=e.response.text)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
 # Add a route for Prometheus metrics (optional, if not using a separate Prometheus server)
 @app.get("/metrics")
 async def metrics():