Language selection checkboxes & filtering in backend

evals/backend.py

@@ -3,11 +3,12 @@ import json
 import numpy as np
 import pandas as pd
 import uvicorn
-from fastapi import FastAPI
+from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
+
 from languages import languages
 from models import models
 from tables import aggregate, make_country_table, make_language_table, make_model_table
@@ -18,7 +19,7 @@ app.add_middleware(CORSMiddleware, allow_origins=["*"])
 app.add_middleware(GZipMiddleware, minimum_size=1000)
 
 with open("results.json", "r") as f:
-    results = json.load(f)
+    results = pd.DataFrame(json.load(f))
 
 
 def serialize(df):
@@ -26,8 +27,17 @@ def serialize(df):
 
 
 @app.post("/api/data")
-def data():
-    _, lang_results, model_results, task_results = aggregate(results)
+async def data(request: Request):
+    body = await request.body()
+    data = json.loads(body)
+    selected_languages = data.get("selectedLanguages", {})
+    df = results
+    if selected_languages:
+        df = df[df["bcp_47"].isin(l["bcp_47"] for l in selected_languages)]
+
+    _, lang_results, model_results, task_results = aggregate(df)
+    print(lang_results)
+    # lang_results = pd.merge(languages, lang_results, on="bcp_47", how="outer")
     model_table = make_model_table(model_results, models)
     language_table = make_language_table(lang_results, languages)
     datasets_df = pd.read_json("data/datasets.json")

evals/main.py

@@ -1,10 +1,8 @@
 import asyncio
 import json
 
-import numpy as np
 from languages import languages
 from models import model_fast, models
-from rich import print
 from tasks import tasks
 from tqdm.asyncio import tqdm_asyncio
 
@@ -39,6 +37,7 @@ async def evaluate():
 
 async def main():
     results = await evaluate()
+    results = [r for group in results for r in group]
     with open("results.json", "w") as f:
         json.dump(results, f, indent=2, ensure_ascii=False)
 

evals/models.py

@@ -96,6 +96,7 @@ models = pd.DataFrame(models, columns=["id"])
 
 api = HfApi()
 
+@cache
 def get_metadata(id):
     try:
         info = api.model_info(id)

evals/tables.py

@@ -5,7 +5,6 @@ make_country_table = make_country_table
 
 
 def aggregate(results):
-    results = pd.DataFrame([r for rs in results for r in rs])
     results = (
         results.groupby(["model", "bcp_47", "task", "metric"]).mean().reset_index()
     )
@@ -14,7 +13,6 @@ def aggregate(results):
         .agg({"score": "mean", "model": "nunique"})
         .reset_index()
     )
-    # lang_results = pd.merge(languages, lang_results, on="bcp_47", how="outer")
     model_results = (
         results.groupby(["model", "task", "metric"])
         .agg({"score": "mean", "bcp_47": "nunique"})
@@ -32,14 +30,6 @@ def mean(lst):
     return sum(lst) / len(lst) if lst else None
 
 
-def fmt_name(s):
-    return (
-        " ".join(w.capitalize() for w in s.split("-"))
-        .replace("Gpt", "GPT")
-        .replace("ai", "AI")
-    )
-
-
 def make_model_table(df, models):
     df["task_metric"] = df["task"] + "_" + df["metric"]
     df = df.drop(columns=["task", "metric"])
@@ -51,13 +41,10 @@ def make_model_table(df, models):
         df[row] = df[row].round(2)
     df = pd.merge(df, models, left_on="model", right_on="id", how="left")
     df["creation_date"] = df["creation_date"].dt.strftime("%Y-%m-%d")
-    df["provider"] = df["model"].str.split("/").str[0].apply(fmt_name)
-    df["model"] = df["model"].str.split("/").str[1].apply(fmt_name)
     df["rank"] = df.index + 1
     df = df[
         [
             "rank",
-            "provider",
             "model",
             "hf_id",
             "creation_date",

frontend/src/App.js

@@ -15,20 +15,11 @@ function App () {
   const [loading, setLoading] = useState(true)
   const [error, setError] = useState(null)
   const [allSuggestions, setAllSuggestions] = useState([])
-  const [languageFilters, setLanguageFilters] = useState({
-    language_name: { value: null, matchMode: FilterMatchMode.EQUALS }, // via global search
-    family: { value: null, matchMode: FilterMatchMode.IN },
-    speakers: { value: null, matchMode: FilterMatchMode.BETWEEN }
-  })
-  const [modelFilters, setModelFilters] = useState({
-    model: { value: null, matchMode: FilterMatchMode.ENDS_WITH }, // via global search
-    type: { value: null, matchMode: FilterMatchMode.IN },
-    size: { value: null, matchMode: FilterMatchMode.BETWEEN }
-  })
+  const [selectedLanguages, setSelectedLanguages] = useState([])
   useEffect(() => {
     fetch('/api/data', {
       method: 'POST',
-      // body: JSON.stringify({ modelFilters, languageFilters }),
+      body: JSON.stringify({ selectedLanguages })
     })
       .then(response => {
         if (!response.ok) {
@@ -44,7 +35,7 @@ function App () {
         setError(err.message)
         setLoading(false)
       })
-  }, [modelFilters, languageFilters])
+  }, [selectedLanguages])
 
   useEffect(() => {
     if (data) {
@@ -52,14 +43,15 @@ function App () {
         type: 'Model',
         value: item.model,
         detail: item.provider,
-        searchText: item.provider.toLowerCase() + ' ' + item.model.toLowerCase()
+        searchText: item.model.toLowerCase()
       }))
       const languages = data.language_table.map(item => ({
         type: 'Language',
         value: item.autonym,
         detail: item.language_name,
         searchText:
-          item.language_name.toLowerCase() + ' ' + item.autonym.toLowerCase()
+          item.language_name.toLowerCase() + ' ' + item.autonym.toLowerCase(),
+        bcp_47: item.bcp_47
       }))
       const datasets = data.dataset_table.map(item => ({
         type: 'Dataset',
@@ -107,25 +99,10 @@ function App () {
           <AutoComplete
             allSuggestions={allSuggestions}
             onComplete={item => {
-              const languageValue =
-                item.type === 'Language' ? item.detail : null
-              const modelValue = item.type === 'Model' ? item.value : null
-              setLanguageFilters(prevFilters => ({
-                ...prevFilters,
-                language_name: {
-                  value: languageValue,
-                  matchMode: FilterMatchMode.EQUALS
-                }
-              }))
-              setModelFilters(prevFilters => ({
-                ...prevFilters,
-                model: {
-                  value: modelValue,
-                  matchMode: FilterMatchMode.ENDS_WITH
-                }
-              }))
+              if (item.type === 'Language') setSelectedLanguages(() => [item])
             }}
           />
+          <pre>{JSON.stringify(selectedLanguages, null, 2)}</pre>
         </header>
         <main
           style={{
@@ -152,11 +129,7 @@ function App () {
                   maxWidth: 'min(100vw, 800px)'
                 }}
               >
-                <ModelTable
-                  data={data.model_table}
-                  filters={modelFilters}
-                  setFilters={setModelFilters}
-                />
+                <ModelTable data={data.model_table} />
               </div>
               <div
                 style={{
@@ -166,8 +139,8 @@ function App () {
               >
                 <LanguageTable
                   data={data.language_table}
-                  filters={languageFilters}
-                  setFilters={setLanguageFilters}
+                  selectedLanguages={selectedLanguages}
+                  setSelectedLanguages={setSelectedLanguages}
                 />
               </div>
               <div

frontend/src/components/AutoComplete.js

@@ -5,7 +5,6 @@ const AutoComplete = ({ allSuggestions, onComplete }) => {
   const [suggestions, setSuggestions] = useState([])
 
   const search = e => {
-    console.log(allSuggestions)
     const matches = allSuggestions.filter(suggestion =>
       suggestion.searchText.includes(e.query.toLowerCase())
     )

frontend/src/components/LanguageTable.js

@@ -6,7 +6,13 @@ import { useState, useEffect } from 'react'
 import { Slider } from 'primereact/slider'
 import ScoreField from './ScoreField'
 
-const LanguageTable = ({ data, filters, setFilters }) => {
+const LanguageTable = ({ data, selectedLanguages, setSelectedLanguages }) => {
+  const [filters, setFilters] = useState({
+    language_name: { value: null, matchMode: FilterMatchMode.EQUALS }, // via global search
+    family: { value: null, matchMode: FilterMatchMode.IN },
+    speakers: { value: null, matchMode: FilterMatchMode.BETWEEN }
+  })
+
   const families = [...new Set(data.map(item => item.family))].slice(0, 10)
   families.push("Other")
   const familyRowFilterTemplate = options => {
@@ -124,10 +130,15 @@ const LanguageTable = ({ data, filters, setFilters }) => {
       removableSort
       filters={filters}
       filterDisplay='menu'
+      selectionMode='checkbox'
+      selection={selectedLanguages}
+      onSelectionChange={e => setSelectedLanguages(e.value)}
+      frozenValue={selectedLanguages}
       scrollable
       scrollHeight='600px'
       id='language-table'
     >
+      <Column selectionMode="multiple" headerStyle={{ width: '3rem' }} />
       <Column
         field='language_name'
         header='Language'

frontend/src/components/ModelTable.js

@@ -7,7 +7,11 @@ import Medal from './Medal'
 import { Slider } from 'primereact/slider'
 import ScoreField from './ScoreField'
 
-const ModelTable = ({ data, filters, setFilters }) => {
+const ModelTable = ({ data }) => {
+  const [filters, setFilters] = useState({
+    type: { value: null, matchMode: FilterMatchMode.IN },
+    size: { value: null, matchMode: FilterMatchMode.BETWEEN }
+  })
   const rankBodyTemplate = rowData => {
     return <Medal rank={rowData.rank} />
   }
@@ -99,9 +103,17 @@ const ModelTable = ({ data, filters, setFilters }) => {
     return <div style={{ textAlign: 'center' }}>{sizeStr}</div>
   }
 
+  const capitalize = s => String(s).charAt(0).toUpperCase() + String(s).slice(1)
+
+  const providerBodyTemplate = rowData => {
+    const providerName = rowData.model.split('/')[0].split('-').map(capitalize).join(' ')
+    return providerName
+  }
+
   const modelBodyTemplate = rowData => {
+    const modelName = rowData.model.split('/')[1].split('-').map(capitalize).join(' ')
     return (
-      <div style={{ fontWeight: 'bold', height: '100%' }}>{rowData.model}</div>
+      <div style={{ fontWeight: 'bold', height: '100%' }}>{modelName}</div>
     )
   }
 
@@ -135,7 +147,7 @@ const ModelTable = ({ data, filters, setFilters }) => {
       id='model-table'
     >
       <Column field='rank' body={rankBodyTemplate} />
-      <Column field='provider' header='Provider' style={{ minWidth: '5rem' }} />
+      <Column field='provider' header='Provider' style={{ minWidth: '5rem' }} body={providerBodyTemplate} />
       <Column
         field='model'
         header='Model'