Upload 4 files

src/leaderboard_utils.py

@@ -5,11 +5,11 @@ import json
 from io import StringIO
 from datetime import datetime
 
-from src.assets.text_content import REPO
+from src.assets.text_content import REPO, BENCHMARK_FILE
 
 def get_github_data():
     """
-    Read and process data from CSV files hosted on GitHub. - https://github.com/clembench/clembench-runs
+    Read and process data from CSV files hosted on GitHub. - https://github.com/clembench/clembench-runs (REPO)
     Set the path in src/assets/text_content/REPO
 
     Returns:
@@ -18,74 +18,60 @@ def get_github_data():
             - "multimodal": List of DataFrames for each version's multimodal leaderboard data.
             - "date": Formatted date of the latest version in "DD Month YYYY" format.
     """
-    base_repo = REPO
-    json_url = base_repo + "benchmark_runs.json"
+    json_url = REPO + BENCHMARK_FILE
     response = requests.get(json_url)
 
     # Check if the JSON file request was successful
     if response.status_code != 200:
-        print(f"Failed to read JSON file: Status Code: {response.status_code}")
+        print(f"Failed to read JSON file - {BENCHMARK_FILE} in repo {REPO}: Status Code: {response.status_code}")
         return None, None, None, None
 
     json_data = response.json()
     versions = json_data['versions']
 
+    # Sort the versions in benchmark by latest first
     version_names = sorted(
         [ver['version'] for ver in versions],
         key=lambda v: list(map(int, v[1:].split('_')[0].split('.'))),  
         reverse=True
     )   
 
-    # Get Leaderboard data - for text-only + multimodal
-    github_data = {}
-
-    # Collect Dataframes
-    text_dfs = []
-    mm_dfs = []
-
-    text_flag = True
-    text_date = ""
-    mm_flag = True
-    mm_date = ""
+    # Collect Dataframes - Text and Multimodal Only - Ignoring _quantized, _backends, _ascii
+    text_data = {
+        'version_data': [],
+        'dataframes': []
+    }
+    multimodal_data = {
+        'version_data': [],
+        'dataframes': []
+    }
 
     for version in version_names:
-        # Collect CSV data in descending order of clembench-runs versions
-        # Collect Text-only data
-        if len(version.split('_')) == 1: 
-            text_url = f"{base_repo}{version}/results.csv"
-            csv_response = requests.get(text_url)
-            if csv_response.status_code == 200:
-                df = pd.read_csv(StringIO(csv_response.text))
-                df = process_df(df)
-                df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-                text_dfs.append(df)
-                if text_flag:
-                    text_flag = False
-                    text_date = next(ver['last_updated'] for ver in versions if ver['version'] == version)
-                    text_date = datetime.strptime(text_date, "%Y-%m-%d").strftime("%d %b %Y")  
+        results_url = f"{REPO}{version}/results.csv"
+        csv_response = requests.get(results_url)
+        if csv_response.status_code == 200:
+            df = pd.read_csv(StringIO(csv_response.text))
+            df = process_df(df)
+            df = df.sort_values(by=df.columns[1], ascending=False) # Sort by Clemscore
 
+            version_data = {
+                'name': version,
+                'last_updated': [datetime.strptime(v['last_updated'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version],
+                'release_date': [datetime.strptime(v['release_date'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version]
+            } 
+
+            if 'multimodal' in version:
+                multimodal_data['dataframes'].append(df)
+                multimodal_data['version_data'].append(version_data)
             else:
-                print(f"Failed to read Text-only leaderboard CSV file for version: {version}. Status Code: {csv_response.status_code}")
-
-        # Check if version ends with 'multimodal' before constructing the URL
-        mm_suffix = "_multimodal" if not version.endswith('multimodal') else ""
-        mm_url = f"{base_repo}{version}{mm_suffix}/results.csv" 
-        mm_response = requests.get(mm_url)
-        if mm_response.status_code == 200:
-            df = pd.read_csv(StringIO(mm_response.text))
-            df = process_df(df)
-            df = df.sort_values(by=df.columns[1], ascending=False) # Sort by clemscore column
-            mm_dfs.append(df)
-            if mm_flag:
-                mm_flag = False
-                mm_date = next(ver['last_updated'] for ver in versions if ver['version'] == version)
-                mm_date = datetime.strptime(mm_date, "%Y-%m-%d").strftime("%d %b %Y")
+                text_data['dataframes'].append(df)
+                text_data['version_data'].append(version_data)
 
       
-    github_data["text"] = text_dfs
-    github_data["multimodal"] = mm_dfs
-    github_data["date"] = text_date
-    github_data["mm_date"] = mm_date
+    github_data = {
+        'text': text_data,
+        'multimodal': multimodal_data
+    }
 
     return github_data
 
@@ -145,3 +131,7 @@ def query_search(df: pd.DataFrame, query: str) -> pd.DataFrame:
 
     return filtered_df
 
+if __name__=='__main__':
+    data = get_github_data()
+    print(data['text']['version_data'])
+    print(data['multimodal']['version_data'])

src/plot_utils.py

@@ -173,7 +173,7 @@ def update_open_models(leaderboard: str = TEXT_NAME):
         Updated checkbox group for Open Models, based on the leaderboard selected
     """
     github_data = get_github_data()
-    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"]['dataframes'][0]
     models = leaderboard_data.iloc[:, 0].unique().tolist()
     open_models, commercial_models = split_models(models)
     return gr.CheckboxGroup(
@@ -193,7 +193,7 @@ def update_closed_models(leaderboard: str = TEXT_NAME):
         Updated checkbox group for Closed Models, based on the leaderboard selected
     """
     github_data = get_github_data()
-    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    leaderboard_data = github_data["text" if leaderboard == TEXT_NAME else "multimodal"]['dataframes'][0]
     models = leaderboard_data.iloc[:, 0].unique().tolist()
     open_models, commercial_models = split_models(models)
     return gr.CheckboxGroup(
@@ -212,7 +212,7 @@ def get_plot_df(leaderboard: str = TEXT_NAME) -> pd.DataFrame:
         DataFrame with model data.
     """
     github_data = get_github_data()
-    return github_data["text" if leaderboard == TEXT_NAME else "multimodal"][0]
+    return github_data["text" if leaderboard == TEXT_NAME else "multimodal"]['dataframes'][0]
 
 
 """

src/trend_utils.py

@@ -79,10 +79,10 @@ def populate_list(df: pd.DataFrame, abs_diff: float) -> list:
             prev_clemscore = curr_clemscore
             prev_date = curr_date
 
-    # Add the last model if the difference between the last and previous date is greater than 15 days
-    last_date = df.iloc[-1]['release_date']
-    if date_difference(last_date, prev_date) > 15:
-        l.append(df.iloc[-1]['model'])
+    # # Add the last model if the difference between the last and previous date is greater than 15 days
+    # last_date = df.iloc[-1]['release_date']
+    # if date_difference(last_date, prev_date) > 15:
+    #     l.append(df.iloc[-1]['model'])
 
     return l
 
@@ -335,11 +335,14 @@ def get_final_trend_plot(benchmark: str = "Text", mobile_view: bool = False) ->
     else:
         height = 1000
 
-    plot_kwargs = {'height': height, 'open_dip': -0.5, 'comm_dip': -5,
+    plot_kwargs = {'height': height, 'open_dip': 0, 'comm_dip': 0,
                    'mobile_view': mobile_view}
 
+    # plot_kwargs = {'height': height, 'open_dip': -0.5, 'comm_dip': -5,
+    #                'mobile_view': mobile_view}
+
     if benchmark == "Text":
-        text_dfs = get_github_data()['text']
+        text_dfs = get_github_data()['text']['dataframes']
         text_result_df = get_trend_data(text_dfs, model_registry_data)
 
         ## Get benchmark tickvalues as dates for X-axis
@@ -349,7 +352,7 @@ def get_final_trend_plot(benchmark: str = "Text", mobile_view: bool = False) ->
                 benchmark_ticks[pd.to_datetime(ver['release_date'])] = ver['version']
         fig =  get_plot(text_result_df, start_date=START_DATE, end_date=datetime.now().strftime('%Y-%m-%d'), benchmark_ticks=benchmark_ticks, **plot_kwargs)
     else:
-        mm_dfs = get_github_data()['multimodal']
+        mm_dfs = get_github_data()['multimodal']['dataframes']
         result_df = get_trend_data(mm_dfs, model_registry_data)
         df = result_df
 
@@ -357,9 +360,12 @@ def get_final_trend_plot(benchmark: str = "Text", mobile_view: bool = False) ->
         benchmark_ticks = {}
         for ver in versions:
             if 'multimodal' in ver['version']:
-                ver['version'] = ver['version'].replace('_multimodal', '')
-            if date_difference(ver['release_date'], '2024-07-15') >= 0:
-                benchmark_ticks[pd.to_datetime(ver['release_date'])] = ver['version'] ## MM benchmark dates considered after v1.6 (incl.)
+                temp_ver = ver['version']
+                temp_ver = temp_ver.replace('_multimodal', '')
+                benchmark_ticks[pd.to_datetime(ver['release_date'])] = temp_ver ## MM benchmark dates considered after v1.6 (incl.)
+
+        print("benchmark_ticks")
+        print(benchmark_ticks)
         fig = get_plot(df, start_date=START_DATE, end_date=datetime.now().strftime('%Y-%m-%d'), benchmark_ticks=benchmark_ticks, **plot_kwargs)
 
     return fig

src/version_utils.py

@@ -9,18 +9,20 @@ import json
 from io import StringIO
 
 from src.leaderboard_utils import process_df
-from src.assets.text_content import REPO
+from src.assets.text_content import REPO, BENCHMARK_FILE
 
-def get_versions_data():
+VARIANTS = ['ascii', 'backends', 'quantized'] # Include other variants if added in the main clembench-runs repo
+
+def get_version_data():
     """
     Read and process data from CSV files of all available versions hosted on GitHub. - https://github.com/clembench/clembench-runs
 
     Returns:
-        versions_data:
+        version_data:
             -
     """
     base_repo = REPO
-    json_url = base_repo + "benchmark_runs.json"
+    json_url = base_repo + BENCHMARK_FILE
     response = requests.get(json_url)
 
     # Check if the JSON file request was successful
@@ -37,51 +39,44 @@ def get_versions_data():
         reverse=True
     )   
 
-    # Get Last updated date of the latest version
-    latest_version = version_names[0]
-    latest_date = next(
-        ver['last_updated'] for ver in versions if ver['version'] == latest_version
-    )
-    formatted_date = datetime.strptime(latest_date, "%Y-%m-%d").strftime("%d %b %Y") 
-
-    # Get Versions data
-    versions_data = {"latest": latest_version, "date": formatted_date}
-
-    # Collect Dataframes
-    dfs = []
+    version_data  = {
+        'versions': [],
+        'dataframes': []
+    }
 
     for version in version_names:
-        text_url = f"{base_repo}{version}/results.csv"
-        mm_url = f"{base_repo}{version}_multimodal/results.csv"
-        quant_url = f"{base_repo}{version}_quantized/results.csv"
-
-        # Text Data
-        response = requests.get(text_url)
+        base_url = f"{base_repo}{version}/results.csv"
+        response = requests.get(base_url)
         if response.status_code == 200:
             df = pd.read_csv(StringIO(response.text))
             df = process_df(df)
             df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version] = df
-
-        # Multimodal Data
-        mm_response = requests.get(mm_url)
-        if mm_response.status_code == 200:
-            mm_df = pd.read_csv(StringIO(mm_response.text))
-            mm_df = process_df(mm_df)
-            mm_df = mm_df.sort_values(by=mm_df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version+"_multimodal"] = mm_df
-
-        # Multimodal Data
-        q_response = requests.get(quant_url)
-        if q_response.status_code == 200:
-            q_df = pd.read_csv(StringIO(q_response.text))
-            q_df = process_df(q_df)
-            q_df = q_df.sort_values(by=q_df.columns[1], ascending=False)  # Sort by clemscore column
-            versions_data[version + "_quantized"] = q_df
-
-    return versions_data
+            version_data['dataframes'].append(df)
+            metadata = {
+                'name': version,
+                'last_updated': [datetime.strptime(v['last_updated'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version],
+                'release_date': [datetime.strptime(v['release_date'], '%Y-%m-%d').strftime("%d %b %Y") for v in versions if v['version'] == version]
+            } 
+            version_data['versions'].append(metadata)
+
+        # Look for variant results file
+        version = version.split('_')[0] # Remove _multimodal suffix, and check for other suffixes
+        for suffix in VARIANTS:
+            base_url = f"{base_repo}{version}_{suffix}/results.csv"
+            response = requests.get(base_url)
+            if response.status_code == 200:
+                df = pd.read_csv(StringIO(response.text))
+                df = process_df(df)
+                df = df.sort_values(by=df.columns[1], ascending=False)  # Sort by clemscore column
+                version_data['dataframes'].append(df)
+                metadata = {
+                    'name': version + "_" + suffix # Skip Release date and last updated # Not included in becnhmark_runs.json
+                } 
+                version_data['versions'].append(metadata)
+
+    return version_data
 
 
 if __name__ == "__main__":
-    versions_data = get_versions_data()
-    print(versions_data.keys())
+    version_data = get_version_data()
+    print(version_data['versions'])