big df

app.py

@@ -20,20 +20,7 @@ def change_version(version):
     return new_df
 
 # Function to create plots
-def create_plots(df, selected_methods):
-    if not selected_methods:
-        return plt.figure()  # Return an empty plot if no method is selected
-
-    filtered_df = df[df['Method'].isin(selected_methods)]
-    fig, ax = plt.subplots()
-    for method in selected_methods:
-        method_df = filtered_df[filtered_df['Method'] == method]
-        ax.plot(method_df['PPL'], label=method)  # Example: Plotting PPL, replace with your metrics
-
-    ax.set_xlabel('Index')  # Modify as per your data
-    ax.set_ylabel('PPL')  # Modify as per your data
-    ax.legend()
-    return fig
+from plotter import create_plots
 
 # Initialize Gradio app
 demo = gr.Blocks()

app_bkp.py

@@ -1,59 +0,0 @@
-import gradio as gr
-import pandas as pd
-
-# Function to load data from a given CSV file
-def load_data(version):
-    file_path = f'versions/{version}.csv'  # Assuming filenames are version1.csv, version2.csv, version3.csv
-    return pd.read_csv(file_path)
-
-# Function for searching in the leaderboard
-def search_leaderboard(df, query):
-    if query == "":
-        return df
-    else:
-        return df[df['Method'].str.contains(query)]
-
-# Function to change the version of the leaderboard
-def change_version(version):
-    new_df = load_data(version)
-    return new_df
-
-# Initialize Gradio app
-demo = gr.Blocks()
-
-with demo:
-    gr.Markdown("## 🥇 TOFU Leaderboard")
-
-    with gr.Row():
-        version_dropdown = gr.Dropdown(
-            choices=["llama", "phi", "stable-lm"],
-            label="🔄 Select Base Model",
-            value="llama",
-        )
-
-    with gr.Row():
-        search_bar = gr.Textbox(
-            placeholder="Search for methods...",
-            show_label=False,
-        )
-
-    leaderboard_table = gr.components.Dataframe(
-        value=load_data("llama"),  # Load initial version (version llama)
-        interactive=True,
-        visible=True,
-    )
-
-    version_dropdown.change(
-        change_version,
-        inputs=version_dropdown,
-        outputs=leaderboard_table
-    )
-
-    search_bar.change(
-        search_leaderboard,
-        inputs=[leaderboard_table, search_bar],
-        outputs=leaderboard_table
-    )
-
-# Launch the app
-demo.launch()

app_old.py

@@ -1,128 +0,0 @@
-import gradio as gr
-
-from src.assets.text_content import TITLE, INTRODUCTION_TEXT
-from src.utils import get_data, compare_plots, filter_search
-
-############################ For Leaderboards #############################
-DATA_PATH = 'versions'
-latest_flag = True #Set flag to iclude latest data inz Details and Versions Tab
-latest_df, latest_vname, previous_df, previous_vname = get_data(DATA_PATH, latest_flag)
-
-global prev_df
-prev_df = previous_df[0]
-def select_prev_df(name):
-    ind = previous_vname.index(name)
-    prev_df = previous_df[ind]
-    return prev_df
-
-############################ For Plots ####################################
-global plot_df, MODEL_COLS
-plot_df = latest_df[0]
-MODEL_COLS = list(plot_df['Model'].unique())
-
-
-############# MAIN APPLICATION ######################
-demo = gr.Blocks()
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🥇 TOFU Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                search_bar = gr.Textbox(
-                    placeholder=" 🔍 Search for models - separate multiple queries with `;` and press ENTER...",
-                    show_label=False,
-                    elem_id="search-bar",
-                )
-                        
-            leaderboard_table = gr.components.Dataframe(
-                value=latest_df[0],
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            # Add a dummy leaderboard to handle search queries from the latest_df and not update latest_df
-            dummy_leaderboard_table = gr.components.Dataframe(
-                value=latest_df[0],
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=False,
-            )
-                
-            search_bar.submit(
-                filter_search,
-                [dummy_leaderboard_table, search_bar],
-                leaderboard_table,
-                queue=True
-            )
-        with gr.TabItem("📈 Plot", id=3):
-            with gr.Row():
-                model_cols = gr.CheckboxGroup(
-                    MODEL_COLS, 
-                    label="Select Models 🤖", 
-                    value=[],
-                    elem_id="column-select",
-                    interactive=True,
-                )
-
-            with gr.Row():
-                plot_grdf = gr.DataFrame(
-                    value=plot_df,
-                    visible=False
-                )
-            with gr.Row():
-                # Output block for the plot
-                plot_output = gr.Plot()
-
-            model_cols.change(
-                compare_plots,
-                [plot_grdf, model_cols],
-                plot_output,
-                queue=True
-            )
-
-        with gr.TabItem("🔄 Versions and Details", elem_id="details", id=2):
-            with gr.Row():
-                ver_selection = gr.Dropdown(
-                    previous_vname, label="Select Version 🕹️", value=previous_vname[0]
-                )
-            with gr.Row():
-                search_bar_prev = gr.Textbox(
-                    placeholder=" 🔍 Search for models - separate multiple queries with `;` and press ENTER...",
-                    show_label=False,
-                    elem_id="search-bar-2",
-                )
-
-            prev_table = gr.components.Dataframe(
-                value=prev_df,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )
-
-            dummy_prev_table = gr.components.Dataframe(
-                value=prev_df,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=False,
-            )
-
-            search_bar_prev.submit(
-                filter_search,
-                [dummy_prev_table, search_bar_prev],
-                prev_table,
-                queue=True
-            )
-
-            ver_selection.change(
-                select_prev_df,
-                [ver_selection],
-                prev_table,
-                queue=True
-            )
-
-    demo.load()
-demo.queue()
-demo.launch()

plotter.py

@@ -0,0 +1,80 @@
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+import scipy.stats as stats
+
+import warnings
+warnings.simplefilter("ignore", category=Warning)
+
+def custom_agg(x):
+    result = stats.hmean(x)
+    return result
+
+def create_plots(big_df, selected_methods):
+    big_df = big_df[big_df['Method'].isin(selected_methods)]
+    # we want 1-Rouge-P
+    big_df["ROUGE-P Forget"] = 1 - big_df["ROUGE-P Forget"]
+
+    metrics = list(big_df.columns)
+    metrics.remove("Method")
+    metrics.remove("Model")
+    metrics.remove("Forget Rate")
+    metrics.remove("LR")
+    metrics.remove("Epoch")
+    metrics.remove("Compute")
+
+    print(metrics)
+    # Apply the custom aggregation function across each row, excluding the first column
+    row_custom_agg = big_df.iloc[:, -len(metrics):].apply(custom_agg, axis=1)
+
+    # If you want to add these results back to your original DataFrame
+    big_df['MAPO'] = row_custom_agg
+    big_df["LR"] = big_df["LR"].astype(float)
+    # big_df = big_df[big_df["LR"] >= 1e-5]
+    big_df["ROUGE-P Forget"] = 1 - big_df["ROUGE-P Forget"]
+
+    big_df.reset_index(inplace=True)
+    print(big_df[["Method", "Model", "Forget Rate", "LR", "Epoch", "ROUGE-P Forget", "MAPO"]].round(2).to_markdown())
+
+    # print(big_df.groupby(['Method', 'Model', 'Forget Rate']).head())
+    result = big_df.loc[big_df.groupby(['Method', 'Model', 'Forget Rate'])['MAPO'].idxmax()]
+    print(result[["Method", "Model", "Forget Rate", "LR", "Epoch", "MAPO"]].round(6).to_markdown())
+    # exit()
+
+    plot_legend = False
+    fs = 18 if plot_legend else 22
+    metrics.append("MAPO")
+
+    # Set the style of the visualization
+    sns.set_theme(style="whitegrid")
+    plt.rcParams['font.family'] = 'Times New Roman'
+
+    for metric_to_plot in metrics:
+        sub_df = result[big_df["Model"] == "Llama-2-7B"]
+        fig, ax = plt.subplots(figsize=(15, 5))
+        sns.barplot(x="Method", y=metric_to_plot, hue="Forget Rate", data=sub_df, ax=ax, legend=plot_legend)
+        ax.set_ylabel(metric_to_plot, fontsize=fs)
+        ax.set_ylim(0.0, 1.0)
+        ax.set_xlabel("", fontsize=fs)
+        ax.set_xticklabels(ax.get_xticklabels(), fontsize=fs)
+        ax.set_yticklabels(ax.get_yticklabels(), fontsize=fs-4)
+        ax.spines[['right', 'top']].set_visible(False)
+        if plot_legend:
+            plt.legend(loc='upper left', bbox_to_anchor=(1.05, 1), title="Forget Rate (%)")
+        plt.title(metric_to_plot + " on Llama-2-7B", fontsize=fs)
+        plt.tight_layout()
+        plt.savefig(f"barplots/{metric_to_plot}-Llama-2-7B{'legend' if plot_legend else ''}.pdf")
+        print(f"\includegraphics[width=\\textwidth]{{figures/barplots/{metric_to_plot}-Llama-2-7B{'legend' if plot_legend else ''}.pdf}}")
+        plt.close(fig)
+
+    for model in ["Llama-2-7B", "Phi"]:
+        sub_df = result[result["Model"] == model][["Method", "Forget Rate", "MAPO"]]
+        # print(sub_df.round(6).to_latex(index=False))
+        sub_df.reset_index(inplace=True)
+
+        # Reorienting the dataframe
+        sub_df_reoriented = sub_df.pivot(index="Method", columns='Forget Rate', values='MAPO')
+
+        # Output a latex table of the MAPO values by Method and Forget Rate
+        print(sub_df_reoriented.round(4).to_latex(index=True))