Update app.py

app.py

@@ -7,16 +7,13 @@ from datetime import datetime
 import re
 
 # Load the Bambara ASR dataset
-print("Loading dataset...")
 dataset = load_dataset("sudoping01/bambara-asr-benchmark", name="default")["train"]
 references = {row["id"]: row["text"] for row in dataset}
 
-# Load or initialize the leaderboard
+# Initialize leaderboard file if it doesn't exist
 leaderboard_file = "leaderboard.csv"
 if not os.path.exists(leaderboard_file):
     pd.DataFrame(columns=["submitter", "WER", "CER", "timestamp"]).to_csv(leaderboard_file, index=False)
-else:
-    print(f"Loaded existing leaderboard with {len(pd.read_csv(leaderboard_file))} entries")
 
 def normalize_text(text):
     """
@@ -28,132 +25,66 @@ def normalize_text(text):
     """
     if not isinstance(text, str):
         text = str(text)
-    
-    # Convert to lowercase
     text = text.lower()
-    
-    # Remove punctuation, keeping spaces
     text = re.sub(r'[^\w\s]', '', text)
-    
-    # Normalize whitespace
     text = re.sub(r'\s+', ' ', text).strip()
-    
     return text
 
 def calculate_metrics(predictions_df):
-    """Calculate WER and CER for predictions."""
+    """Calculate WER and CER for predictions against reference dataset."""
     results = []
-
     for _, row in predictions_df.iterrows():
         id_val = row["id"]
         if id_val not in references:
-            print(f"Warning: ID {id_val} not found in references")
             continue
-            
         reference = normalize_text(references[id_val])
         hypothesis = normalize_text(row["text"])
-        
-        # Print detailed info for first few entries
-        if len(results) < 5:
-            print(f"ID: {id_val}")
-            print(f"Reference: '{reference}'")
-            print(f"Hypothesis: '{hypothesis}'")
-        
-        # Skip empty strings
         if not reference or not hypothesis:
-            print(f"Warning: Empty reference or hypothesis for ID {id_val}")
             continue
-            
-        # Split into words for jiwer
-        reference_words = reference.split()
-        hypothesis_words = hypothesis.split()
-        
-        if len(results) < 5:
-            print(f"Reference words: {reference_words}")
-            print(f"Hypothesis words: {hypothesis_words}")
-        
-        # Calculate metrics
         try:
-            # Make sure we're not comparing identical strings
-            if reference == hypothesis:
-                print(f"Warning: Identical strings for ID {id_val}")
-                # Force a small difference if the strings are identical
-                # This is for debugging - remove in production if needed
-                if len(hypothesis_words) > 0:
-                    # Add a dummy word to force non-zero WER
-                    hypothesis_words.append("dummy_debug_token")
-                    hypothesis = " ".join(hypothesis_words)
-            
-            # Calculate WER and CER
             sample_wer = wer(reference, hypothesis)
             sample_cer = cer(reference, hypothesis)
-            
-            if len(results) < 5:
-                print(f"WER: {sample_wer}, CER: {sample_cer}")
-                
             results.append({
                 "id": id_val,
-                "reference": reference,
-                "hypothesis": hypothesis,
                 "wer": sample_wer,
                 "cer": sample_cer
             })
-        except Exception as e:
-            print(f"Error calculating metrics for ID {id_val}: {str(e)}")
-    
+        except Exception:
+            pass  # Skip invalid samples silently
     if not results:
-        raise ValueError("No valid samples for WER/CER calculation")
-        
-    # Calculate average metrics
+        raise ValueError("No valid samples available for metric calculation")
     avg_wer = sum(item["wer"] for item in results) / len(results)
     avg_cer = sum(item["cer"] for item in results) / len(results)
-    
     return avg_wer, avg_cer, results
 
 def process_submission(submitter_name, csv_file):
+    """Process the uploaded CSV, calculate metrics, and update the leaderboard."""
     try:
-        # Read and validate the uploaded CSV
         df = pd.read_csv(csv_file)
-        print(f"Processing submission from {submitter_name} with {len(df)} rows")
-        
         if len(df) == 0:
-            return "Error: Uploaded CSV is empty.", None
-            
+            return "Submission failed: The uploaded CSV file is empty. Please upload a valid CSV file with predictions.", None
         if set(df.columns) != {"id", "text"}:
-            return f"Error: CSV must contain exactly 'id' and 'text' columns. Found: {', '.join(df.columns)}", None
-            
+            return f"Submission failed: The CSV file must contain exactly two columns: 'id' and 'text'. Found: {', '.join(df.columns)}", None
         if df["id"].duplicated().any():
-            dup_ids = df[df["id"].duplicated()]["id"].unique()
-            return f"Error: Duplicate IDs found: {', '.join(map(str, dup_ids[:5]))}", None
-
-        # Check if IDs match the reference dataset
+            dup_ids = df[df["id"].duplicated(keep=False)]["id"].unique()
+            return f"Submission failed: Duplicate 'id' values detected: {', '.join(map(str, dup_ids[:5]))}", None
         missing_ids = set(references.keys()) - set(df["id"])
         extra_ids = set(df["id"]) - set(references.keys())
-        
         if missing_ids:
-            return f"Error: Missing {len(missing_ids)} IDs in submission. First few missing: {', '.join(map(str, list(missing_ids)[:5]))}", None
-            
+            return f"Submission failed: Missing {len(missing_ids)} required 'id' values. First few: {', '.join(map(str, list(missing_ids)[:5]))}", None
         if extra_ids:
-            return f"Error: Found {len(extra_ids)} extra IDs not in reference dataset. First few extra: {', '.join(map(str, list(extra_ids)[:5]))}", None
+            return f"Submission failed: Found {len(extra_ids)} unrecognized 'id' values. First few: {', '.join(map(str, list(extra_ids)[:5]))}", None
+        empty_ids = [row["id"] for _, row in df.iterrows() if not normalize_text(row["text"])]
+        if empty_ids:
+            return f"Submission failed: Empty transcriptions detected for {len(empty_ids)} 'id' values. First few: {', '.join(map(str, empty_ids[:5]))}", None
         
-        # Calculate WER and CER
-        try:
-            avg_wer, avg_cer, detailed_results = calculate_metrics(df)
-            
-            # Debug information
-            print(f"Calculated metrics - WER: {avg_wer:.4f}, CER: {avg_cer:.4f}")
-            print(f"Processed {len(detailed_results)} valid samples")
-            
-            # Check for suspiciously low values
-            if avg_wer < 0.001:
-                print("WARNING: WER is extremely low - likely an error")
-                return "Error: WER calculation yielded suspicious results (near-zero). Please check your submission CSV.", None
-                
-        except Exception as e:
-            print(f"Error in metrics calculation: {str(e)}")
-            return f"Error calculating metrics: {str(e)}", None
+        # Calculate metrics
+        avg_wer, avg_cer, detailed_results = calculate_metrics(df)
+        n_valid = len(detailed_results)
+        if n_valid == 0:
+            return "Submission failed: No valid samples found for metric calculation.", None
         
-        # Update the leaderboard
+        # Update leaderboard
         leaderboard = pd.read_csv(leaderboard_file)
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         new_entry = pd.DataFrame(
@@ -163,46 +94,74 @@ def process_submission(submitter_name, csv_file):
         leaderboard = pd.concat([leaderboard, new_entry]).sort_values("WER")
         leaderboard.to_csv(leaderboard_file, index=False)
         
-        return f"Submission processed successfully! WER: {avg_wer:.4f}, CER: {avg_cer:.4f}", leaderboard
+        # Format leaderboard for display
+        display_leaderboard = leaderboard.copy()
+        display_leaderboard["WER"] = display_leaderboard["WER"].apply(lambda x: f"{x:.4f}")
+        display_leaderboard["CER"] = display_leaderboard["CER"].apply(lambda x: f"{x:.4f}")
         
+        return f"Your submission has been successfully processed. Evaluated {n_valid} valid samples. WER: {avg_wer:.4f}, CER: {avg_cer:.4f}", display_leaderboard
+    
     except Exception as e:
-        print(f"Error processing submission: {str(e)}")
-        return f"Error processing submission: {str(e)}", None
+        return f"Submission failed: An error occurred while processing your file - {str(e)}", None
 
-# Create the Gradio interface
-with gr.Blocks(title="Bambara ASR Leaderboard") as demo:
+def load_and_format_leaderboard():
+    """Load the leaderboard and format WER/CER for display."""
+    if os.path.exists(leaderboard_file):
+        leaderboard = pd.read_csv(leaderboard_file)
+        leaderboard["WER"] = leaderboard["WER"].apply(lambda x: f"{x:.4f}")
+        leaderboard["CER"] = leaderboard["CER"].apply(lambda x: f"{x:.4f}")
+        return leaderboard
+    return pd.DataFrame(columns=["submitter", "WER", "CER", "timestamp"])
+
+# Gradio interface
+with gr.Blocks(title="Bambara ASR Benchmark Leaderboard") as demo:
     gr.Markdown(
         """
-        # Bambara ASR Leaderboard
-        Upload a CSV file with 'id' and 'text' columns to evaluate your ASR predictions.
-        The 'id's must match those in the dataset.
-        [View the dataset here](https://huggingface.co/datasets/MALIBA-AI/bambara_general_leaderboard_dataset).
-        - **WER**: Word Error Rate (lower is better).
-        - **CER**: Character Error Rate (lower is better).
+        ## Bambara ASR Benchmark Leaderboard
+
+        **Welcome to the Bambara Automatic Speech Recognition (ASR) Benchmark Leaderboard**  
+        Evaluate your ASR model's performance on the Bambara language dataset.
+
+        ### Submission Instructions
+        1. Prepare a CSV file with two columns:  
+           - **`id`**: Must match identifiers in the official dataset.  
+           - **`text`**: Your model's transcription predictions.  
+        2. Ensure the CSV file meets these requirements:  
+           - Contains only `id` and `text` columns.  
+           - No duplicate `id` values.  
+           - All `id` values match dataset entries.  
+        3. Upload your CSV file below.
+
+        ### Dataset
+        Access the official dataset: [Bambara ASR Dataset](https://huggingface.co/datasets/MALIBA-AI/bambara_general_leaderboard_dataset)
+
+        ### Evaluation Metrics
+        - **Word Error Rate (WER)**: Word-level transcription accuracy (lower is better).  
+        - **Character Error Rate (CER)**: Character-level accuracy (lower is better).
+
+        ### Leaderboard
+        Submissions are ranked by WER and include:  
+        - Submitter name  
+        - WER (4 decimal places)  
+        - CER (4 decimal places)  
+        - Submission timestamp  
         """
     )
-    
     with gr.Row():
-        submitter = gr.Textbox(label="Submitter Name or Model Name", placeholder="e.g., MALIBA-AI/asr")
-        csv_upload = gr.File(label="Upload CSV File", file_types=[".csv"])
-        
-    submit_btn = gr.Button("Submit")
-    output_msg = gr.Textbox(label="Status", interactive=False)
+        submitter = gr.Textbox(label="Submitter Name or Model Identifier", placeholder="e.g., MALIBA-AI/asr")
+        csv_upload = gr.File(label="Upload Prediction CSV File", file_types=[".csv"])
+    submit_btn = gr.Button("Evaluate Submission")
+    output_msg = gr.Textbox(label="Submission Status", interactive=False)
     leaderboard_display = gr.DataFrame(
-        label="Leaderboard",
-        value=pd.read_csv(leaderboard_file),
+        label="Current Leaderboard",
+        value=load_and_format_leaderboard(),
         interactive=False
     )
-    
     submit_btn.click(
         fn=process_submission,
         inputs=[submitter, csv_upload],
         outputs=[output_msg, leaderboard_display]
     )
 
-# Print startup message
-print("Starting Bambara ASR Leaderboard app...")
-
-# Launch the app
 if __name__ == "__main__":
     demo.launch(share=True)