add csv data indexing

main.py

@@ -6,6 +6,8 @@ import json
 import os
 import logging
 from txtai.embeddings import Embeddings
+import pandas as pd
+import glob
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -105,6 +107,41 @@ async def query_index(request: QueryRequest):
         logger.error(f"Error querying index: {str(e)}")
         raise HTTPException(status_code=500, detail=f"Error querying index: {str(e)}")
 
+def process_csv_file(file_path):
+    try:
+        df = pd.read_csv(file_path)
+        df_rows = df.apply(lambda row: ' '.join(row.values.astype(str)), axis=1)
+        txtai_data = [(i, row, None) for i, row in enumerate(df_rows)]
+        return txtai_data, df_rows.tolist()
+    except Exception as e:
+        logger.error(f"Error processing CSV file {file_path}: {str(e)}")
+        return None, None
+
+def check_and_index_csv_files():
+    index_data_folder = "/app/index_data"
+    if not os.path.exists(index_data_folder):
+        logger.warning(f"index_data folder not found: {index_data_folder}")
+        return
+
+    csv_files = glob.glob(os.path.join(index_data_folder, "*.csv"))
+    for csv_file in csv_files:
+        index_id = os.path.splitext(os.path.basename(csv_file))[0]
+        if not os.path.exists(f"/app/indexes/{index_id}"):
+            logger.info(f"Processing CSV file: {csv_file}")
+            txtai_data, documents = process_csv_file(csv_file)
+            if txtai_data and documents:
+                embeddings.index(txtai_data)
+                save_embeddings(index_id, documents)
+                logger.info(f"CSV file indexed successfully: {csv_file}")
+            else:
+                logger.warning(f"Failed to process CSV file: {csv_file}")
+        else:
+            logger.info(f"Index already exists for: {csv_file}")
+
+@app.on_event("startup")
+async def startup_event():
+    check_and_index_csv_files()
+
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=8000)