update with examples and save to dataset

app.py

@@ -7,20 +7,31 @@ def gradio_run_evaluation(inference_api, model_name, prompt_format):
         output.append(result)
         yield "\n".join(output)
 
-with gr.Blocks() as demo:
+with gr.Blocks(gr.themes.Soft()) as demo:
     gr.Markdown("# DuckDB SQL Evaluation App")
 
     inference_api = gr.Dropdown(
         label="Inference API",
-        choices=['openrouter', 'inference_api'],
+        choices=['openrouter'],
         value="openrouter"
     )
     model_name = gr.Textbox(label="Model Name (e.g., qwen/qwen-2.5-72b-instruct)")
+    gr.Markdown("[View OpenRouter Models](https://openrouter.ai/models?order=top-weekly)")
+
     prompt_format = gr.Dropdown(
         label="Prompt Format",
         choices=['duckdbinst', 'duckdbinstgraniteshort'], #AVAILABLE_PROMPT_FORMATS,
         value="duckdbinstgraniteshort"
     )
+    gr.Examples(
+        examples=[
+            ["openrouter", "qwen/qwen-2.5-72b-instruct", "duckdbinst"],
+            ["openrouter", "meta-llama/llama-3.2-3b-instruct:free", "duckdbinstgraniteshort"],
+            ["openrouter", "mistralai/mistral-nemo", "duckdbinst"],
+        ],
+        inputs=[inference_api, model_name, prompt_format],
+    )
+
     start_btn = gr.Button("Start Evaluation")
     output = gr.Textbox(label="Output", lines=20)
 

evaluation_logic.py

@@ -4,14 +4,14 @@ from pathlib import Path
 from datetime import datetime
 import json
 import traceback
+import uuid
+from huggingface_hub import CommitScheduler
 
-# Add the necessary directories to the Python path
 current_dir = Path(__file__).resolve().parent
 duckdb_nsql_dir = current_dir / 'duckdb-nsql'
 eval_dir = duckdb_nsql_dir / 'eval'
 sys.path.extend([str(current_dir), str(duckdb_nsql_dir), str(eval_dir)])
 
-# Import necessary functions and classes
 from eval.predict import get_manifest, DefaultLoader, PROMPT_FORMATTERS, generate_sql
 from eval.evaluate import evaluate, compute_metrics, get_to_print
 from eval.evaluate import test_suite_evaluation, read_tables_json
@@ -19,6 +19,54 @@ from eval.schema import TextToSQLParams, Table
 
 AVAILABLE_PROMPT_FORMATS = list(PROMPT_FORMATTERS.keys())
 
+prediction_folder = Path("prediction_results/")
+evaluation_folder = Path("evaluation_results/")
+
+file_uuid = uuid.uuid4()
+
+prediction_scheduler = CommitScheduler(
+    repo_id="sql-console/duckdb-nsql-predictions",
+    repo_type="dataset",
+    folder_path=prediction_folder,
+    path_in_repo="data",
+    every=10,
+)
+
+evaluation_scheduler = CommitScheduler(
+    repo_id="sql-console/duckdb-nsql-scores",
+    repo_type="dataset",
+    folder_path=evaluation_folder,
+    path_in_repo="data",
+    every=10,
+)
+
+def save_prediction(inference_api, model_name, prompt_format, question, generated_sql):
+    prediction_file = prediction_folder / f"prediction_{file_uuid}.json"
+    prediction_folder.mkdir(parents=True, exist_ok=True)
+    with prediction_scheduler.lock:
+        with prediction_file.open("a") as f:
+            json.dump({
+                "inference_api": inference_api,
+                "model_name": model_name,
+                "prompt_format": prompt_format,
+                "question": question,
+                "generated_sql": generated_sql,
+                "timestamp": datetime.now().isoformat()
+            }, f)
+
+def save_evaluation(inference_api, model_name, prompt_format, metrics):
+    evaluation_file = evaluation_folder / f"evaluation_{file_uuid}.json"
+    evaluation_folder.mkdir(parents=True, exist_ok=True)
+    with evaluation_scheduler.lock:
+        with evaluation_file.open("a") as f:
+            json.dump({
+                "inference_api": inference_api,
+                "model_name": model_name,
+                "prompt_format": prompt_format,
+                "metrics": metrics,
+                "timestamp": datetime.now().isoformat()
+            }, f)
+
 def run_prediction(inference_api, model_name, prompt_format, output_file):
     dataset_path = str(eval_dir / "data/dev.json")
     table_meta_path = str(eval_dir / "data/tables.json")
@@ -60,9 +108,6 @@ def run_prediction(inference_api, model_name, prompt_format, output_file):
             else:
                 table_params = []
 
-            #if len(table_params) == 0:
-                #yield f"[red] WARNING: No tables found for {db_id} [/red]"
-
             text_to_sql_inputs.append(TextToSQLParams(
                 instruction=question,
                 database=db_id,
@@ -73,7 +118,7 @@ def run_prediction(inference_api, model_name, prompt_format, output_file):
         generated_sqls = generate_sql(
             manifest=manifest,
             text_to_sql_in=text_to_sql_inputs,
-            retrieved_docs=[[] for _ in text_to_sql_inputs],  # Assuming no retrieved docs
+            retrieved_docs=[[] for _ in text_to_sql_inputs],
             prompt_formatter=prompt_formatter,
             stop_tokens=stop_tokens,
             overwrite_manifest=overwrite_manifest,
@@ -84,12 +129,16 @@ def run_prediction(inference_api, model_name, prompt_format, output_file):
         )
 
         # Save results
+        output_file.parent.mkdir(parents=True, exist_ok=True)
         with output_file.open('w') as f:
             for original_data, (sql, _) in zip(data, generated_sqls):
                 output = {**original_data, "pred": sql}
                 json.dump(output, f)
                 f.write('\n')
 
+                # Save prediction to dataset
+                save_prediction(inference_api, model_name, prompt_format, original_data["question"], sql)
+
         yield f"Prediction completed. Results saved to {output_file}"
     except Exception as e:
         yield f"Prediction failed with error: {str(e)}"
@@ -161,6 +210,9 @@ def run_evaluation(inference_api, model_name, prompt_format="duckdbinstgranitesh
             categories=categories,
         )
 
+        # Save evaluation results to dataset
+        save_evaluation(inference_api, model_name, prompt_format, metrics)
+        
         yield "Evaluation completed."
 
         if metrics: