improvements

app.py

@@ -1,8 +1,9 @@
 import asyncio
-from typing import Optional
+
 import gradio as gr
 import pandas as pd
-from data_access import get_pool, get_async_connection, close_pool
+from data_access import get_pool, get_async_connection, close_pool, get_questions, get_source_finders, get_run_ids, \
+    get_source_runs, get_baseline_rankers, calculate_baseline_vs_source_stats_for_question, get_unified_sources
 
 # Initialize data at the module level
 questions = []
@@ -10,61 +11,26 @@ source_finders = []
 questions_dict = {}
 source_finders_dict = {}
 question_options = []
+baseline_rankers = []
+baseline_rankers_dict = {}
+baseline_ranker_options = []
 run_ids = []
 finder_options = []
 finder_labels = {"All": "All Source Finders"}
 
 
 # Get all questions
-async def get_questions():
-    async with get_async_connection() as conn:
-        questions = await conn.fetch("SELECT id, question_text FROM questions ORDER BY id")
-        return [{"id": q["id"], "text": q["question_text"]} for q in questions]
-
-
-# Get distinct source finders
-async def get_source_finders():
-    async with get_async_connection() as conn:
-        finders = await conn.fetch("SELECT id, source_finder_type as name FROM source_finders ORDER BY id")
-        return [{"id": f["id"], "name": f["name"]} for f in finders]
-
-
-# Get distinct run IDs for a question
-async def get_run_ids(question_id: int):
-    async with get_async_connection() as conn:
-        query = "SELECT DISTINCT run_id FROM source_runs WHERE question_id = $1 order by run_id desc"
-        params = [question_id]
-        run_ids = await conn.fetch(query, *params)
-        return [r["run_id"] for r in run_ids]
-
-
-# Get source runs for a specific question with filters
-async def get_source_runs(question_id: int, source_finder_id: Optional[int] = None,
-                          run_id: Optional[int] = None):
-    async with get_async_connection() as conn:
-        # Build query with filters
-        query = """
-            SELECT sr.*, sf.source_finder_type as finder_name
-            FROM source_runs sr
-            JOIN source_finders sf ON sr.source_finder_id = sf.id
-            WHERE sr.question_id = $1 and sr.run_id = $2
-            AND sr.source_finder_id = $3
-        """
-        params = [question_id, run_id, source_finder_id]
-
-        query += " ORDER BY sr.rank DESC"
-
-        sources = await conn.fetch(query, *params)
-        return [dict(s) for s in sources]
-
 
 # Initialize data in a single async function
 async def initialize_data():
-    global questions, source_finders, questions_dict, source_finders_dict, question_options, finder_options, finder_labels
+    global questions, source_finders, questions_dict, source_finders_dict, question_options, finder_options, finder_labels, baseline_rankers, baseline_ranker_options
 
     questions = await get_questions()
     source_finders = await get_source_finders()
 
+    baseline_rankers = await get_baseline_rankers()
+    baseline_rankers_dict = {f["id"]: f["name"] for f in baseline_rankers}
+
     # Convert to dictionaries for easier lookup
     questions_dict = {q["id"]: q["text"] for q in questions}
     source_finders_dict = {f["id"]: f["name"] for f in source_finders}
@@ -73,10 +39,12 @@ async def initialize_data():
     question_options = [f"{q['id']}: {q['text']}" for q in questions]
     finder_options = [str(f["id"]) for f in source_finders]
     finder_labels = {str(f["id"]): f["name"] for f in source_finders}
+    baseline_ranker_options = [f["id"] for f in baseline_rankers]
+    baseline_ranker_labels = {str(f["id"]): f["name"] for f in source_finders}
 
 
 # Main function to handle UI interactions
-def update_source_runs(question_option, source_finder_id, run_id):
+def update_sources_list(question_option, source_finder_id, baseline_ranker_id: str, run_id:str):
     if not question_option:
         return None, [], "No question selected", None
 
@@ -86,35 +54,36 @@ def update_source_runs(question_option, source_finder_id, run_id):
     # Get run_ids for filtering - use asyncio.run for each independent operation
     available_run_ids = asyncio.run(get_run_ids(question_id))
     run_id_options = [str(r_id) for r_id in available_run_ids]
+    if run_id not in run_id_options:
+        run_id = run_id_options[0]
 
-    # If the selected run_id is not in available options, reset it
-    # if run_id not in run_id_options:
-    #     run_id = None
-    #
-    # # Convert run_id to int if not "All"
-    run_id_int = available_run_ids[0]
+    run_id_int = int(run_id)
     finder_id_int = None if len(source_finder_id) == 0 else int(source_finder_id)
+    if type(baseline_ranker_id) == list:
+        baseline_ranker_id = baseline_ranker_id[0]
+    baseline_ranker_id_int = 1 if len(baseline_ranker_id) == 0 else int(baseline_ranker_id)
 
+    source_runs = None
+    stats = None
     # Get source runs data
-    source_runs = asyncio.run(get_source_runs(question_id, finder_id_int, run_id_int))
+    if finder_id_int:
+        source_runs, stats = asyncio.run(get_unified_sources(question_id, finder_id_int, run_id_int, baseline_ranker_id_int))
+        # Create DataFrame for display
+        df = pd.DataFrame(source_runs)
 
     if not source_runs:
-        return None, run_id_options, "No results found for the selected filters"
+        return None, None, run_id_options, "No results found for the selected filters",
 
-    # Create DataFrame for display
-    df = pd.DataFrame(source_runs)
 
     # Format table columns
-    columns_to_display = ['finder_name', 'run_id', 'sugya_id', 'tractate', 'folio', 'rank', 'reason']
+    columns_to_display = ['sugya_id', 'in_baseline', 'baseline_rank', 'in_source_run', 'source_run_rank', 'tractate', 'folio', 'reason']
     df_display = df[columns_to_display] if all(col in df.columns for col in columns_to_display) else df
 
     # CSV for download
     # csv_data = df.to_csv(index=False)
 
     result_message = f"Found {len(source_runs)} results"
-
-    return df_display, run_id_options, result_message,
-
+    return df_display, stats, run_id_options, result_message,
 
 
 # Create Gradio app
@@ -128,31 +97,52 @@ async def main():
 
         with gr.Row():
             with gr.Column(scale=3):
-                # Main content area
-                question_dropdown = gr.Dropdown(
-                    choices=question_options,
-                    label="Select Question",
-                    value=None,
-                    interactive=True
-                )
-
-                run_id_dropdown = gr.Dropdown(
-                    choices=run_ids,
-                    value="1",
-                    allow_custom_value=True,
-                    label="Run ids for Question",
-                    interactive=True
-                )
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        # Main content area
+                        question_dropdown = gr.Dropdown(
+                            choices=question_options,
+                            label="Select Question",
+                            value=None,
+                            interactive=True
+                        )
+                    with gr.Column(scale=1):
+                        baseline_rankers_dropdown = gr.Dropdown(
+                            choices=baseline_ranker_options,
+                            label="Select Baseline Ranker",
+                            interactive=True
+                        )
 
                 with gr.Row():
-                    source_finder_dropdown = gr.Dropdown(
-                        choices=finder_options,
-                        label="Source Finder",
-                        interactive=True
-                    )
+                    with gr.Column(scale=1):
+                        source_finder_dropdown = gr.Dropdown(
+                            choices=finder_options,
+                            label="Source Finder",
+                            interactive=True
+                        )
+                    with gr.Column(scale=1):
+                        run_id_dropdown = gr.Dropdown(
+                            choices=run_ids,
+                            value="1",
+                            allow_custom_value=True,
+                            label="Run id for Question and source finder",
+                            interactive=True
+                        )
 
-                result_text = gr.Markdown("Select a question to view source runs")
 
+                result_text = gr.Markdown("Select a question to view source runs")
+                gr.Markdown("# Source Run Statistics")
+                statistics_table = gr.DataFrame(
+                    headers=["num_high_ranked_baseline_sources",
+                            "num_high_ranked_found_sources",
+                            "overlap_count",
+                            "overlap_percentage",
+                            "high_ranked_overlap_count",
+                            "high_ranked_overlap_percentage"
+                    ],
+                    interactive=False,
+                )
+                gr.Markdown("# Sources Found")
                 results_table = gr.DataFrame(
                     headers=['Source Finder', 'Run ID', 'Sugya ID', 'Tractate', 'Folio', 'Rank', 'Reason'],
                     interactive=False
@@ -177,37 +167,40 @@ async def main():
                 gr.Markdown("### Source Finders")
                 for f in source_finders:
                     gr.Markdown(f"**{f['id']}**: {f['name']}")
+                gr.Markdown("### Baseline Source Rankers")
+                for f in baseline_rankers:
+                    gr.Markdown(f"**{f['id']}**: {f['name']}")
 
         # Set up event handlers
         question_dropdown.change(
-            update_source_runs,
-            inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown],
+            update_sources_list,
+            inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown, baseline_rankers_dropdown],
             # outputs=[run_id_dropdown, results_table, result_text, download_button]
-            outputs=[results_table, run_id_dropdown, result_text]
+            outputs=[results_table, statistics_table, run_id_dropdown, result_text]
         )
 
         source_finder_dropdown.change(
-            update_source_runs,
-            inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown],
+            update_sources_list,
+            inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown, baseline_rankers_dropdown],
             # outputs=[run_id_dropdown, results_table, result_text, download_button]
-            outputs=[results_table, run_id_dropdown, result_text]
+            outputs=[results_table, statistics_table, run_id_dropdown, result_text]
         )
 
         run_id_dropdown.change(
-             update_source_runs,
-             inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown],
-             outputs=[results_table, run_id_dropdown, result_text]
+             update_sources_list,
+             inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown, baseline_rankers_dropdown],
+             outputs=[results_table, statistics_table, run_id_dropdown, result_text]
         )
 
         # Initial load of data when question is selected
         question_dropdown.change(
-            update_source_runs,
-            inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown],
-            outputs=[results_table, run_id_dropdown, result_text]
+            update_sources_list,
+            inputs=[question_dropdown, source_finder_dropdown, run_id_dropdown, baseline_rankers_dropdown],
+            outputs=[results_table, statistics_table, run_id_dropdown, result_text]
         )
 
     app.queue()
     app.launch()
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(main())

data_access.py

@@ -1,9 +1,12 @@
 import asyncio
 import os
 from contextlib import asynccontextmanager
+from typing import Optional
 
 import asyncpg
+import psycopg2
 from dotenv import load_dotenv
+import pandas as pd
 
 # Global connection pool
 _pool = None
@@ -53,3 +56,148 @@ async def close_pool():
         await _pool.close()
         _pool = None
 
+
+async def get_questions():
+    async with get_async_connection() as conn:
+        questions = await conn.fetch("SELECT id, question_text FROM questions ORDER BY id")
+        return [{"id": q["id"], "text": q["question_text"]} for q in questions]
+
+
+# Get distinct source finders
+async def get_source_finders():
+    async with get_async_connection() as conn:
+        finders = await conn.fetch("SELECT id, source_finder_type as name FROM source_finders ORDER BY id")
+        return [{"id": f["id"], "name": f["name"]} for f in finders]
+
+
+# Get distinct run IDs for a question
+async def get_run_ids(question_id: int):
+    async with get_async_connection() as conn:
+        query = "SELECT DISTINCT run_id FROM source_runs WHERE question_id = $1 order by run_id desc"
+        params = [question_id]
+        run_ids = await conn.fetch(query, *params)
+        return [r["run_id"] for r in run_ids]
+
+
+# Get source runs for a specific question with filters
+async def get_source_runs(question_id: int, source_finder_id: Optional[int] = None,
+                          run_id: Optional[int] = None):
+    async with get_async_connection() as conn:
+        # Build query with filters
+        query = """
+            SELECT sr.*, sf.source_finder_type as finder_name
+            FROM source_runs sr
+            JOIN source_finders sf ON sr.source_finder_id = sf.id
+            WHERE sr.question_id = $1 and sr.run_id = $2
+            AND sr.source_finder_id = $3
+        """
+        params = [question_id, run_id, source_finder_id]
+
+        query += " ORDER BY sr.rank DESC"
+
+        sources = await conn.fetch(query, *params)
+        return [dict(s) for s in sources]
+
+async def get_baseline_rankers():
+    async with get_async_connection() as conn:
+        rankers = await conn.fetch("SELECT id, ranker FROM rankers ORDER BY id")
+        return [{"id": f["id"], "name": f["ranker"]} for f in rankers]
+
+async def calculate_baseline_vs_source_stats_for_question(baseline_sources , source_runs_sources):
+    # for a given question_id and source_finder_id and run_id calculate the baseline vs source stats
+    # e.g. overlap, high ranked overlap, etc.
+    async with get_async_connection() as conn:
+        actual_sources_set = {s["sugya_id"] for s in source_runs_sources}
+        baseline_sources_set = {s["sugya_id"] for s in baseline_sources}
+
+        # Calculate overlap
+        overlap = actual_sources_set.intersection(baseline_sources_set)
+        # only_in_1 = actual_sources_set - baseline_sources_set
+        # only_in_2 = baseline_sources_set - actual_sources_set
+
+        # Calculate high-ranked overlap (rank >= 4)
+        actual_high_ranked = {s["sugya_id"] for s in source_runs_sources if int(s["source_rank"]) >= 4}
+        baseline_high_ranked = {s["sugya_id"] for s in baseline_sources if int(s["baseline_rank"]) >= 4}
+
+        high_ranked_overlap = actual_high_ranked.intersection(baseline_high_ranked)
+
+        results = {
+            "total_baseline_sources": len(baseline_sources),
+            "total_found_sources": len(source_runs_sources),
+            "overlap_count": len(overlap),
+            "overlap_percentage": round(len(overlap) * 100 / max(len(actual_sources_set), len(baseline_sources_set)),
+                                        2) if max(len(actual_sources_set), len(baseline_sources_set)) > 0 else 0,
+            "num_high_ranked_baseline_sources": len(baseline_high_ranked),
+            "num_high_ranked_found_sources": len(actual_high_ranked),
+            "high_ranked_overlap_count": len(high_ranked_overlap),
+            "high_ranked_overlap_percentage": round(len(high_ranked_overlap) * 100 / max(len(actual_high_ranked), len(baseline_high_ranked)), 2) if max(len(actual_high_ranked), len(baseline_high_ranked)) > 0 else 0
+        }
+        #convert results to dataframe
+        results_df = pd.DataFrame([results])
+        return results_df
+
+
+async def get_unified_sources(question_id: int, source_finder_id: int, run_id: int, ranker_id: int):
+    """
+    Create unified view of sources from both baseline_sources and source_runs
+    with indicators of where each source appears and their respective ranks.
+    """
+    async with get_async_connection() as conn:
+        # Get sources from source_runs
+        query_runs = """
+            SELECT sr.sugya_id, sr.rank as source_rank, sr.tractate, sr.folio, sr.reason as source_reason
+            FROM source_runs sr
+            WHERE sr.question_id = $1 AND sr.source_finder_id = $2 AND sr.run_id = $3
+        """
+        source_runs = await conn.fetch(query_runs, question_id, source_finder_id, run_id)
+
+        # Get sources from baseline_sources
+        query_baseline = """
+            SELECT bs.sugya_id, bs.rank as baseline_rank, bs.tractate, bs.folio
+            FROM baseline_sources bs
+            WHERE bs.question_id = $1 AND bs.ranker_id = $2
+        """
+        baseline_sources = await conn.fetch(query_baseline, question_id, ranker_id)
+
+        stats_df = await calculate_baseline_vs_source_stats_for_question(baseline_sources, source_runs)
+
+        # Convert to dictionaries for easier lookup
+        source_runs_dict = {s["sugya_id"]: dict(s) for s in source_runs}
+        baseline_dict = {s["sugya_id"]: dict(s) for s in baseline_sources}
+
+        # Get all unique sugya_ids
+        all_sugya_ids = set(source_runs_dict.keys()) | set(baseline_dict.keys())
+
+        # Build unified results
+        unified_results = []
+        for sugya_id in all_sugya_ids:
+            in_source_run = sugya_id in source_runs_dict
+            in_baseline = sugya_id in baseline_dict
+            if in_baseline:
+                info = baseline_dict[sugya_id]
+            else:
+                info = source_runs_dict[sugya_id]
+            result = {
+                "sugya_id": sugya_id,
+                "tractate": info.get("tractate", "N/A"),
+                "folio": info.get("folio", "N/A"),
+                "in_baseline": "Yes" if in_baseline else "No",
+                "baseline_rank": baseline_dict.get(sugya_id, {}).get("baseline_rank", "N/A"),
+                "in_source_run": "Yes" if in_source_run else "No",
+                "source_run_rank": source_runs_dict.get(sugya_id, {}).get("source_rank", "N/A"),
+                "source_reason": source_runs_dict.get(sugya_id, {}).get("reason", "N/A")
+            }
+            unified_results.append(result)
+
+
+        return unified_results, stats_df
+
+
+def get_pg_sync_connection(schema="talmudexplore"):
+    conn = psycopg2.connect(dbname=os.getenv("pg_dbname"),
+        user=os.getenv("pg_user"),
+        password=os.getenv("pg_password"),
+        host=os.getenv("pg_host"),
+        port=os.getenv("pg_port"),
+        options=f"-c search_path={schema}")
+    return conn

eval_tables.py

@@ -0,0 +1,106 @@
+from data_access import get_pg_sync_connection
+
+conn = get_pg_sync_connection()
+
+
+
+def create_eval_database():
+    """Create SQLite database with a proper relational structure."""
+    # Connect to the database (creates it if it doesn't exist)
+    cursor = conn.cursor()
+
+    # Create questions table
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS questions (
+        id SERIAL PRIMARY KEY,
+        question_text TEXT NOT NULL,
+    CONSTRAINT unique_question_text UNIQUE (question_text)
+);
+    ''')
+
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS rankers (
+            id SERIAL PRIMARY KEY,
+            ranker TEXT NOT NULL        
+    );
+        ''')
+
+    # Create table for unique sources
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS baseline_sources (
+        id SERIAL PRIMARY KEY,
+        question_id INTEGER NOT NULL,
+        tractate TEXT NOT NULL,
+        folio TEXT NOT NULL,
+        sugya_id TEXT NOT NULL,
+        rank INTEGER NOT NULL,
+        reason TEXT,         
+        ranker_id INTEGER NOT NULL,
+        FOREIGN KEY (question_id) REFERENCES questions(id),
+        FOREIGN KEY (ranker_id) REFERENCES rankers(id),
+        CONSTRAINT unique_source_per_question_ranker UNIQUE(question_id, sugya_id, ranker_id)
+    );
+    ''')
+
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS source_finders (
+            id SERIAL PRIMARY KEY,
+            source_finder_type TEXT NOT NULL,
+            description TEXT,
+            source_finder_version TEXT NOT NULL
+        );
+        ''')
+
+    # Create table for logging all sources from each run
+    cursor.execute('''
+    CREATE TABLE IF NOT EXISTS source_runs (
+        id SERIAL PRIMARY KEY,
+        source_finder_id INTEGER NOT NULL,
+        run_id TEXT NOT NULL,
+        run_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+        question_id INTEGER NOT NULL,
+        tractate TEXT NOT NULL,
+        folio TEXT NOT NULL,
+        sugya_id TEXT NOT NULL,
+        rank INTEGER NOT NULL,
+        reason TEXT,
+        FOREIGN KEY (source_finder_id) REFERENCES source_finders(id),
+        FOREIGN KEY (question_id) REFERENCES questions(id)
+    );
+    ''')
+
+    conn.commit()
+    conn.close()
+
+def load_source_finders():
+    cursor = conn.cursor()
+    for item in ["claude_sources", "keywords", "lenses"]:
+        cursor.execute("INSERT INTO source_finders (source_finder_type, source_finder_version) VALUES (%s, 1)", (item,))
+    conn.commit()
+
+def load_rankers():
+    cursor = conn.cursor()
+    for item in ["claude_sources"]:
+        cursor.execute("INSERT INTO rankers (ranker) VALUES (%s)", (item,))
+    conn.commit()
+
+def load_baseline_sources():
+    # copy all claude values where run_id = 1 from source_runs to baseline_sources
+    cursor = conn.cursor()
+    cursor.execute('''
+        INSERT INTO baseline_sources (question_id, tractate, folio, sugya_id, rank, reason, ranker_id)
+        SELECT question_id, tractate, folio, sugya_id, rank, reason, 1
+        FROM source_runs
+        WHERE run_id = 1 and source_finder_id = 1
+    ''')
+    conn.commit()
+
+
+
+if __name__ == '__main__':
+    # Create the database
+    # create_eval_database()
+    load_baseline_sources()
+
+
+

requirements.txt

@@ -1,3 +1,4 @@
 asyncpg
 gradio
-dotenv
+dotenv
+psycopg2

tests/requirements.txt

@@ -0,0 +1,2 @@
+pytest
+pytest-asyncio

tests/test_db_layer.py

@@ -0,0 +1,19 @@
+import pytest
+
+from data_access import get_unified_sources
+
+
+@pytest.mark.asyncio
+async def test_get_unified_sources():
+    results, stats = await get_unified_sources(2, 2, 1, 1)
+    assert results is not None
+    assert stats is not None
+
+    # Check number of rows in results list
+    assert len(results) > 4, "Results should contain at least one row"
+
+    # Check number of rows in stats DataFrame
+    assert stats.shape[0] > 0, "Stats DataFrame should contain at least one row"
+
+    # You can also check specific stats columns
+    assert "overlap_count" in stats.columns, "Stats should contain overlap_count"