refines

app.py

@@ -6,6 +6,8 @@ from functools import lru_cache
 import re
 from collections import Counter
 import editdistance
+import json
+import os
 
 # Cache the dataset loading to avoid reloading on refresh
 @lru_cache(maxsize=1)
@@ -18,6 +20,37 @@ def load_data():
         return load_dataset("parquet", 
                           data_files="https://huggingface.co/datasets/GenSEC-LLM/SLT-Task1-Post-ASR-Text-Correction/resolve/main/data/test-00000-of-00001.parquet")
 
+# Storage for user-submitted methods (in-memory for demo purposes)
+user_methods = []
+
+# Data file for persistence
+USER_DATA_FILE = "user_methods.json"
+
+# Load user methods from file if exists
+def load_user_methods():
+    global user_methods
+    if os.path.exists(USER_DATA_FILE):
+        try:
+            with open(USER_DATA_FILE, 'r') as f:
+                user_methods = json.load(f)
+        except Exception as e:
+            print(f"Error loading user methods: {e}")
+            user_methods = []
+
+# Save user methods to file
+def save_user_methods():
+    try:
+        with open(USER_DATA_FILE, 'w') as f:
+            json.dump(user_methods, f)
+    except Exception as e:
+        print(f"Error saving user methods: {e}")
+
+# Try to load user methods at startup
+try:
+    load_user_methods()
+except:
+    pass
+
 # Preprocess text for better WER calculation
 def preprocess_text(text):
     if not text or not isinstance(text, str):
@@ -353,12 +386,19 @@ def get_wer_metrics(dataset):
     rows.append(nb_oracle_row)
     rows.append(cp_oracle_row)
     
+    # Add user-submitted methods
+    for user_method in user_methods:
+        user_row = {"Methods": user_method["name"]}
+        for source in all_sources + ["OVERALL"]:
+            user_row[source] = user_method.get(source, np.nan)
+        rows.append(user_row)
+    
     # Create DataFrame from rows
     result_df = pd.DataFrame(rows)
     
-    return result_df
+    return result_df, all_sources
 
-# Format the dataframe for display
+# Format the dataframe for display, and sort by performance
 def format_dataframe(df):
     df = df.copy()
     
@@ -378,13 +418,35 @@ def format_dataframe(df):
                 else:
                     df.loc[idx, col] = "N/A"
     
-    return df
+    # Extract the examples row
+    examples_row = df[df["Methods"] == "Number of Examples"]
+    
+    # Get the performance rows
+    performance_rows = df[df["Methods"] != "Number of Examples"]
+    
+    # Convert the OVERALL column to numeric for sorting
+    # First, replace 'N/A' with a high value (worse than any real WER)
+    performance_rows["numeric_overall"] = performance_rows["OVERALL"].replace("N/A", "999")
+    
+    # Convert to float for sorting
+    performance_rows["numeric_overall"] = performance_rows["numeric_overall"].astype(float)
+    
+    # Sort by performance (ascending - lower WER is better)
+    sorted_performance = performance_rows.sort_values(by="numeric_overall")
+    
+    # Drop the numeric column used for sorting
+    sorted_performance = sorted_performance.drop(columns=["numeric_overall"])
+    
+    # Combine the examples row with the sorted performance rows
+    result = pd.concat([examples_row, sorted_performance], ignore_index=True)
+    
+    return result
 
 # Main function to create the leaderboard
 def create_leaderboard():
     dataset = load_data()
-    metrics_df = get_wer_metrics(dataset)
-    return format_dataframe(metrics_df)
+    metrics_df, all_sources = get_wer_metrics(dataset)
+    return format_dataframe(metrics_df), all_sources
 
 # Create the Gradio interface
 with gr.Blocks(title="ASR Text Correction Leaderboard") as demo:
@@ -399,13 +461,56 @@ with gr.Blocks(title="ASR Text Correction Leaderboard") as demo:
     
     with gr.Row():
         try:
-            initial_df = create_leaderboard()
+            initial_df, all_sources = create_leaderboard()
             leaderboard = gr.DataFrame(initial_df)
         except Exception:
             leaderboard = gr.DataFrame(pd.DataFrame([{"Error": "Error initializing leaderboard"}]))
+            all_sources = []
+    
+    gr.Markdown("### Submit Your Method")
+    gr.Markdown("Enter WER values as percentages (e.g., 5.6 for 5.6% WER)")
+    
+    with gr.Row():
+        method_name = gr.Textbox(label="Method Name", placeholder="Enter your method name")
+    
+    # Create input fields for each source
+    source_inputs = {}
+    with gr.Row():
+        with gr.Column():
+            for i, source in enumerate(all_sources):
+                if i < len(all_sources) // 2:
+                    source_inputs[source] = gr.Textbox(label=f"WER for {source}", placeholder="e.g., 5.6")
+        
+        with gr.Column():
+            for i, source in enumerate(all_sources):
+                if i >= len(all_sources) // 2:
+                    source_inputs[source] = gr.Textbox(label=f"WER for {source}", placeholder="e.g., 5.6")
+    
+    with gr.Row():
+        submit_btn = gr.Button("Submit Results")
+    
+    def submit_method(name, **values):
+        if not name:
+            return "Please enter a method name", leaderboard
+        
+        success = add_user_method(name, values)
+        if success:
+            updated_df, _ = create_leaderboard()
+            return "Method added successfully!", updated_df
+        else:
+            return "Error adding method", leaderboard
     
     def refresh_and_report():
-        return create_leaderboard()
+        updated_df, _ = create_leaderboard()
+        return updated_df
+    
+    # Connect buttons to functions
+    submit_args = [method_name] + list(source_inputs.values())
+    submit_btn.click(
+        submit_method, 
+        inputs=[method_name] + list(source_inputs.values()),
+        outputs=[gr.Textbox(label="Status"), leaderboard]
+    )
     
     refresh_btn.click(refresh_and_report, outputs=[leaderboard])