fix: duplicated entries with multiple languages

src/populate.py

@@ -7,30 +7,27 @@ from src.leaderboard.read_evals import get_raw_assessment_results
 
 
 def expand_multi_language_entries(df):
-    """Expand multi-language entries (like 'Python/C++') into separate rows for OR filtering"""
+    """Keep multi-language entries as single rows but create individual language columns for filtering"""
     if df.empty or auto_eval_column_attrs.language.name not in df.columns:
         return df
-        
-    expanded_rows = []
     
-    for idx, row in df.iterrows():
-        lang_value = row[auto_eval_column_attrs.language.name]
-        
-        # If language contains /, create separate rows for each language
-        if isinstance(lang_value, str) and "/" in lang_value:
-            languages = [lang.strip() for lang in lang_value.split("/")]
-            for lang in languages:
-                new_row = row.copy()
-                new_row[auto_eval_column_attrs.language.name] = lang
-                new_row["_original_language"] = lang_value  # Keep original for display
-                expanded_rows.append(new_row)
-        else:
-            # Keep single language rows as is
-            row_copy = row.copy()
-            row_copy["_original_language"] = lang_value
-            expanded_rows.append(row_copy)
+    # Get all unique individual languages
+    all_languages = set()
+    for value in df[auto_eval_column_attrs.language.name].unique():
+        if isinstance(value, str):
+            languages = [lang.strip() for lang in value.split("/")]
+            all_languages.update(languages)
+    
+    # Create individual language columns for filtering
+    for lang in sorted(all_languages):
+        if lang:  # Skip empty strings
+            safe_lang = lang.replace("+", "plus").replace("#", "sharp").replace(" ", "_").lower()
+            col_name = f"_lang_{safe_lang}"
+            df[col_name] = df[auto_eval_column_attrs.language.name].apply(
+                lambda x: lang in str(x) if x is not None else False
+            )
     
-    return pd.DataFrame(expanded_rows).reset_index(drop=True)
+    return df
 
 
 def get_leaderboard_df(eval_results_path, eval_requests_path, cols, benchmark_cols):