Add logging for query results and improve data path handling

app.py

@@ -1,5 +1,7 @@
+import datetime
 import json
 import os
+import uuid
 
 import gradio as gr
 import pandas as pd
@@ -19,6 +21,11 @@ persistent_storage = os.path.exists("/data")
 if persistent_storage:
     # Use persistent storage
     print("Using persistent storage")
+    data_path = "/data"
+else:
+    # Use local storage
+    print("Using local storage")
+    data_path = "./data"
 
 # Embedding model details
 embedding_model_name = "nomadicsynth/research-compass-arxiv-abstracts-embedding-model"
@@ -360,10 +367,29 @@ Return only the JSON object. All key names and string values must be in double q
 # })
 
 
+def log_query_and_results(query_id: str, query: str, results: list[dict]):
+    """Log the query and results to a file."""
+    log_entry = {
+        "timestamp": datetime.now().isoformat(),
+        "query_id": query_id,
+        "query": query,
+        "results": results,
+    }
+    log_file = os.path.join(data_path, "query_results_log.jsonl")
+    with open(log_file, "a") as f:
+        f.write(json.dumps(log_entry) + "\n")
+    
+    # print a short summary of the log entry with timestamp
+    print(f"[{log_entry['timestamp']}] Query ID: {query_id}, Results Count: {len(results)}")
+
+
 def find_synergistic_papers(abstract: str, limit=25) -> list[dict]:
     """Find papers synergistic with the given abstract using FAISS with cosine similarity"""
     global dataset
 
+    # Generate a unique ID for the query
+    query_id = str(uuid.uuid4())
+
     # Normalize the abstract for cosine similarity
     abstract = abstract.replace("\n", " ")
     # Replace multiple whitespaces with a single space
@@ -397,6 +423,9 @@ def find_synergistic_papers(abstract: str, limit=25) -> list[dict]:
         }
         papers.append(paper_dict)
 
+    # Log the query and results
+    log_query_and_results(query_id, abstract, papers)
+
     return papers
 
 
@@ -626,14 +655,15 @@ def create_interface():
         )
 
         # Set up logging directories
-        data_path = "/data" if persistent_storage else "./data"
-        os.makedirs(data_path + "/flagged_paper_matches", exist_ok=True)
-        os.makedirs(data_path + "/flagged_analyses", exist_ok=True)
+        flagged_paper_matches_path = data_path + "/flagged_paper_matches"
+        flagged_analyses_path = data_path + "/flagged_analyses"
+        os.makedirs(flagged_paper_matches_path, exist_ok=True)
+        os.makedirs(flagged_analyses_path, exist_ok=True)
 
         # Set up loggers
         paper_match_logger.setup(
             [abstract_input, paper_details_output, paper_feedback, paper_expert, paper_comment],
-            data_path + "/flagged_paper_matches",
+            flagged_paper_matches_path,
         )
         analysis_logger.setup(
             [
@@ -644,7 +674,7 @@ def create_interface():
                 analysis_expert,
                 analysis_comment,
             ],
-            data_path + "/flagged_analyses",
+            flagged_analyses_path,
         )
 
         # Display paper details when row is selected