add leaderboard

Dockerfile

@@ -0,0 +1,21 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.12
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user . /app/
+
+# COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Run setup
+RUN chmod +x setup.sh
+RUN ./setup.sh
+
+CMD ["streamlit", "run", "app.py", "--server.address=0.0.0.0", "--server.port=7860"]

app.py

@@ -0,0 +1,43 @@
+import streamlit as st
+
+from draw_utils import PAGE_MARKDOWN, PAGE_INFO, LENGTHS
+from draw_utils import load_results, style_dataframe
+
+st.set_page_config(layout="wide", page_title="Leaderboard App")
+st.markdown(PAGE_MARKDOWN, unsafe_allow_html=True)
+
+
+def draw_leaderboard():
+    df = load_results()
+
+    tasks = ['avg'] + [f"qa{i}" for i in range(1, 11)]
+    columns = ["model_name", "avg(32k)", "avg(128k)"] + LENGTHS
+
+    st.title("🔎📚🪡📚❓ BABILong Leaderboard 🏆")
+    st.markdown(PAGE_INFO)
+    st.subheader("Average Accuracy")
+    search_term = st.text_input("Search models:", "")
+
+    tabs = st.tabs([str(task) for task in tasks])
+    for i, tab in enumerate(tabs):
+        with tab:
+            task_df = df[df.task == tasks[i]][columns]
+
+            if search_term:
+                task_df = task_df[task_df['model_name'].str.contains(search_term, case=False)]
+            task_df.reset_index(drop=True, inplace=True)
+
+            row_height = 35
+            height = (len(task_df) + 1) * row_height
+
+            styled = style_dataframe(task_df).format(precision=2)
+
+            st.dataframe(
+                styled,
+                use_container_width=True,
+                height=height,
+            )
+
+
+if __name__ == "__main__":
+    draw_leaderboard()

data/leaderboard-v0_results.csv

@@ -0,0 +1,171 @@
+model_name,task,0k,1k,2k,4k,8k,16k,32k,64k,128k,512k,1M,10M
+GPT-2 (137M),avg,27,15,,,,,,,,,,
+mamba-2.8b-hf,avg,70,52,35,9,0,,,,,,,
+rwkv-6-world-7b,avg,56,55,48,35,7,,,,,,,
+v5-Eagle-7B-HF,avg,62,54,48,41,2,,,,,,,
+Meta-Llama-3-8B-Instruct,avg,64,60,58,50,44,,,,,,,
+LLaMA-2-7B-32K,avg,41,53,45,40,39,32,3,,,,,
+longchat-7b-v1.5-32k,avg,46,42,40,41,42,39,5,,,,,
+LongAlpaca-13B,avg,48,47,46,43,40,36,4,,,,,
+Llama-2-7B-32K-Instruct,avg,49,52,49,43,40,35,5,,,,,
+01-ai/Yi-34B,avg,72,52,43,37,38,31,4,,,,,
+Mistral-7b-Instruct-v0.2,avg,60,56,52,49,45,42,37,,,,,
+Mixtral-8x7B-Instruct-v0.1,avg,65,63,60,55,50,46,40,,,,,
+01-ai/Yi-34B-200k,avg,65,59,56,54,52,50,48,48,,,,
+Mixtral-8x22B-Instruct-v0.1,avg,75,73,70,65,58,51,43,35,,,,
+activation-beacon-llama2-7b-chat,avg,55,52,47,43,36,23,16,8,6,,,
+Yarn-Mistral-7b-128k,avg,51,52,43,40,38,30,16,10,9,,,
+chatglm3-6b-128k,avg,56,55,51,48,46,41,36,21,13,,,
+activation-beacon-mistral-7b,avg,59,56,51,48,43,37,36,27,14,,,
+01-ai/Yi-9B-200k,avg,52,55,48,46,45,36,37,29,24,,,
+Phi-3-mini-128k-instruct,avg,64,57,55,51,50,46,42,37,7,,,
+ai21labs/Jamba-v0.1,avg,65,53,50,48,46,45,41,40,34,,,
+c4ai-command-r-v01,avg,64,64,63,61,59,52,51,46,38,,,
+Meta-Llama-3.1-8B-Instruct,avg,67,68,66,66,62,60,56,49,39,,,
+Phi-3-medium-128k-instruct,avg,72,70,67,62,60,57,53,45,30,,,
+GPT-4,avg,87,81,77,74,71,64,53,43,36,,,
+Meta-Llama-3.1-70B-Instruct,avg,85,81,78,74,70,65,59,53,45,,,
+~ Mamba (130M) fine-tune,avg,,,,98.7,98.5,98.5,98.1,97,92.5,,,
+Llama3-ChatQA-1.5-8B + RAG,avg,48,48,47,46,45,45,44,42,45,42,39,37
+~ RMT (137M) fine-tune,avg,99.36,97.4,94.66,92.32,89.9,85.62,77.88,69.86,58.52,46.36,42.84,33.78
+~ ARMT (137M) fine-tune,avg,99.32,,,98.1,98.2,98.1,98,97.9,96.9,95.3,93.4,76.6
+GPT-2 (137M),qa1,35,13,,,,,,,,,,
+mamba-2.8b-hf,qa1,65,56,40,7,1,,,,,,,
+rwkv-6-world-7b,qa1,65,62,53,39,5,,,,,,,
+v5-Eagle-7B-HF,qa1,68,58,52,36,3,,,,,,,
+Meta-Llama-3-8B-Instruct,qa1,98,93,90,79,62,,,,,,,
+LLaMA-2-7B-32K,qa1,54,57,33,26,34,32,3,,,,,
+longchat-7b-v1.5-32k,qa1,52,60,56,55,50,42,4,,,,,
+LongAlpaca-13B,qa1,58,55,58,50,37,23,2,,,,,
+Llama-2-7B-32K-Instruct,qa1,65,61,52,41,35,23,3,,,,,
+01-ai/Yi-34B,qa1,99,59,51,34,46,31,4,,,,,
+Mistral-7b-Instruct-v0.2,qa1,92,86,75,64,63,57,45,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa1,99,92,84,77,65,53,49,,,,,
+01-ai/Yi-34B-200k,qa1,85,73,68,66,63,65,62,60,,,,
+Mixtral-8x22B-Instruct-v0.1,qa1,100,99,95,89,79,63,40,38,,,,
+activation-beacon-llama2-7b-chat,qa1,85,81,67,65,48,21,16,6,5,,,
+Yarn-Mistral-7b-128k,qa1,61,71,58,45,51,34,21,8,8,,,
+chatglm3-6b-128k,qa1,82,77,74,72,67,56,47,13,13,,,
+activation-beacon-mistral-7b,qa1,92,86,73,59,47,42,42,27,9,,,
+01-ai/Yi-9B-200k,qa1,33,82,69,59,56,47,44,32,29,,,
+Phi-3-mini-128k-instruct,qa1,97,84,72,69,70,60,53,38,1,,,
+ai21labs/Jamba-v0.1,qa1,90,72,66,63,65,58,50,49,38,,,
+c4ai-command-r-v01,qa1,98,95,94,91,89,68,70,50,30,,,
+Phi-3-medium-128k-instruct,qa1,100,93,87,80,81,72,69,58,21,,,
+GPT-4,qa1,99,100,100,95,93,84,62,58,39,,,
+~ Mamba (130M) fine-tune,qa1,100,100,100,100,100,100,100,100,100,92,,
+Llama3-ChatQA-1.5-8B + RAG,qa1,60,62,60,58,58,60,60,56,64,54,55,50
+~ RMT (137M) fine-tune,qa1,100,100,100,100,100,100,99,96,94,87,84,66
+~ ARMT (137M) fine-tune,qa1,100,,,100,100,100,100,100,100,99,99,89
+GPT-2 (137M),qa2,21,17,,,,,,,,,,
+mamba-2.8b-hf,qa2,68,47,24,8,0,,,,,,,
+rwkv-6-world-7b,qa2,42,26,20,16,3,,,,,,,
+v5-Eagle-7B-HF,qa2,43,27,25,19,2,,,,,,,
+Meta-Llama-3-8B-Instruct,qa2,47,46,49,39,20,,,,,,,
+LLaMA-2-7B-32K,qa2,37,40,37,26,27,21,2,,,,,
+longchat-7b-v1.5-32k,qa2,31,19,14,13,18,14,2,,,,,
+LongAlpaca-13B,qa2,30,30,28,29,26,18,2,,,,,
+Llama-2-7B-32K-Instruct,qa2,41,40,36,27,20,16,3,,,,,
+01-ai/Yi-34B,qa2,67,43,32,30,23,15,4,,,,,
+Mistral-7b-Instruct-v0.2,qa2,46,34,32,22,11,9,7,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa2,58,51,46,35,27,18,14,,,,,
+01-ai/Yi-34B-200k,qa2,62,51,46,42,35,32,27,30,,,,
+Mixtral-8x22B-Instruct-v0.1,qa2,77,65,61,56,48,41,33,11,,,,
+activation-beacon-llama2-7b-chat,qa2,52,37,28,25,23,11,9,4,2,,,
+Yarn-Mistral-7b-128k,qa2,47,48,37,30,34,21,12,5,3,,,
+chatglm3-6b-128k,qa2,51,46,45,39,38,31,24,6,5,,,
+activation-beacon-mistral-7b,qa2,45,35,32,28,22,14,12,10,2,,,
+01-ai/Yi-9B-200k,qa2,67,52,43,39,31,25,22,12,8,,,
+Phi-3-mini-128k-instruct,qa2,57,38,38,36,34,23,22,15,2,,,
+ai21labs/Jamba-v0.1,qa2,57,43,42,39,37,29,26,20,16,,,
+c4ai-command-r-v01,qa2,64,58,56,54,50,39,37,32,16,,,
+Phi-3-medium-128k-instruct,qa2,76,62,58,51,44,41,27,14,11,,,
+GPT-4,qa2,88,79,72,68,65,59,42,25,25,,,
+~ Mamba (130M) fine-tune,qa2,98,98,98,98,98,98,98,95,87,,,
+Llama3-ChatQA-1.5-8B + RAG,qa2,28,25,22,19,14,13,9,7,6,6,2,2
+~ RMT (137M) fine-tune,qa2,100,100,99,98,97,94,82,59,39,25,22,19
+~ ARMT (137M) fine-tune,qa2,100,,,100,100,100,100,100,100,99,99,84
+GPT-2 (137M),qa3,6,8,,,,,,,,,,
+mamba-2.8b-hf,qa3,48,39,21,8,0,,,,,,,
+rwkv-6-world-7b,qa3,40,45,28,24,4,,,,,,,
+v5-Eagle-7B-HF,qa3,43,34,30,40,1,,,,,,,
+Meta-Llama-3-8B-Instruct,qa3,33,28,30,26,11,,,,,,,
+LLaMA-2-7B-32K,qa3,32,38,34,28,27,21,1,,,,,
+longchat-7b-v1.5-32k,qa3,22,16,15,17,21,22,4,,,,,
+LongAlpaca-13B,qa3,25,26,26,25,24,27,2,,,,,
+Llama-2-7B-32K-Instruct,qa3,35,36,34,26,23,20,2,,,,,
+01-ai/Yi-34B,qa3,45,34,24,20,17,12,4,,,,,
+Mistral-7b-Instruct-v0.2,qa3,36,34,31,30,24,18,12,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa3,34,32,31,30,27,29,24,,,,,
+01-ai/Yi-34B-200k,qa3,35,30,27,24,24,22,22,26,,,,
+Mixtral-8x22B-Instruct-v0.1,qa3,53,56,49,39,31,27,26,26,,,,
+activation-beacon-llama2-7b-chat,qa3,33,25,25,21,20,17,13,5,5,,,
+Yarn-Mistral-7b-128k,qa3,31,36,33,32,27,25,9,13,7,,,
+chatglm3-6b-128k,qa3,33,37,31,31,27,25,23,17,9,,,
+activation-beacon-mistral-7b,qa3,36,33,25,21,18,15,15,15,16,,,
+01-ai/Yi-9B-200k,qa3,34,33,29,24,25,21,20,20,8,,,
+Phi-3-mini-128k-instruct,qa3,32,41,31,27,26,24,21,22,4,,,
+ai21labs/Jamba-v0.1,qa3,32,31,29,26,24,22,22,21,26,,,
+c4ai-command-r-v01,qa3,25,28,26,28,26,30,28,33,24,,,
+Phi-3-medium-128k-instruct,qa3,53,51,45,35,30,30,27,25,17,,,
+GPT-4,qa3,56,63,57,56,53,45,31,31,32,,,
+~ Mamba (130M) fine-tune,qa3,97,97,97,97,97,96,95,92,81,,,
+Llama3-ChatQA-1.5-8B + RAG,qa3,17,18,17,17,16,17,15,13,19,17,10,11
+~ RMT (137M) fine-tune,qa3,97,94,88,81,73,66,55,55,36,25,22,21
+~ ARMT (137M) fine-tune,qa3,97,,,92,92,92,91,90,86,80,72,37
+GPT-2 (137M),qa4,29,18,,,,,,,,,,
+mamba-2.8b-hf,qa4,96,59,47,12,0,,,,,,,
+rwkv-6-world-7b,qa4,54,65,57,35,7,,,,,,,
+v5-Eagle-7B-HF,qa4,79,74,63,55,3,,,,,,,
+Meta-Llama-3-8B-Instruct,qa4,58,55,50,43,52,,,,,,,
+LLaMA-2-7B-32K,qa4,26,54,51,51,46,36,3,,,,,
+longchat-7b-v1.5-32k,qa4,60,55,52,57,57,49,4,,,,,
+LongAlpaca-13B,qa4,65,61,58,52,50,44,4,,,,,
+Llama-2-7B-32K-Instruct,qa4,39,52,54,56,55,52,6,,,,,
+01-ai/Yi-34B,qa4,59,56,51,55,52,43,4,,,,,
+Mistral-7b-Instruct-v0.2,qa4,54,58,58,60,60,58,54,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa4,55,60,59,61,63,61,58,,,,,
+01-ai/Yi-34B-200k,qa4,64,65,64,63,61,56,54,44,,,,
+Mixtral-8x22B-Instruct-v0.1,qa4,56,62,59,62,62,60,54,39,,,,
+activation-beacon-llama2-7b-chat,qa4,40,50,52,43,34,22,14,9,10,,,
+Yarn-Mistral-7b-128k,qa4,60,56,43,45,32,31,16,7,8,,,
+chatglm3-6b-128k,qa4,45,48,42,38,35,32,27,13,11,,,
+activation-beacon-mistral-7b,qa4,53,58,60,57,53,50,45,29,15,,,
+01-ai/Yi-9B-200k,qa4,49,47,50,50,54,43,45,36,33,,,
+Phi-3-mini-128k-instruct,qa4,54,56,56,50,49,50,45,47,5,,,
+ai21labs/Jamba-v0.1,qa4,64,50,49,49,48,52,46,49,38,,,
+c4ai-command-r-v01,qa4,46,58,59,54,56,46,46,47,52,,,
+Phi-3-medium-128k-instruct,qa4,54,61,63,64,64,61,59,52,33,,,
+GPT-4,qa4,98,70,63,60,52,47,46,40,32,,,
+~ Mamba (130M) fine-tune,qa4,100,100,100,100,100,100,99,100,98,,,
+Llama3-ChatQA-1.5-8B + RAG,qa4,53,58,56,59,57,60,60,59,60,59,54,56
+~ RMT (137M) fine-tune,qa4,100,94,87,83,80,75,64,51,38,26,24,20
+~ ARMT (137M) fine-tune,qa4,100,,,100,100,100,100,100,100,100,100,92
+GPT-2 (137M),qa5,45,19,,,,,,,,,,
+mamba-2.8b-hf,qa5,75,58,43,9,0,,,,,,,
+rwkv-6-world-7b,qa5,79,77,80,61,14,,,,,,,
+v5-Eagle-7B-HF,qa5,75,76,71,57,3,,,,,,,
+Meta-Llama-3-8B-Instruct,qa5,85,78,73,65,73,,,,,,,
+LLaMA-2-7B-32K,qa5,55,74,70,67,59,51,7,,,,,
+longchat-7b-v1.5-32k,qa5,63,62,62,65,66,67,9,,,,,
+LongAlpaca-13B,qa5,63,61,61,61,62,66,12,,,,,
+Llama-2-7B-32K-Instruct,qa5,63,69,69,67,66,63,9,,,,,
+01-ai/Yi-34B,qa5,88,70,59,48,53,55,4,,,,,
+Mistral-7b-Instruct-v0.2,qa5,70,66,66,67,69,67,67,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa5,80,79,80,73,66,67,56,,,,,
+01-ai/Yi-34B-200k,qa5,78,77,77,76,76,75,76,80,,,,
+Mixtral-8x22B-Instruct-v0.1,qa5,87,84,84,79,69,64,63,63,,,,
+activation-beacon-llama2-7b-chat,qa5,65,67,64,63,57,45,29,17,7,,,
+Yarn-Mistral-7b-128k,qa5,58,47,45,47,47,38,23,17,19,,,
+chatglm3-6b-128k,qa5,70,69,64,60,61,61,58,55,26,,,
+activation-beacon-mistral-7b,qa5,68,66,66,74,74,66,67,55,28,,,
+01-ai/Yi-9B-200k,qa5,76,59,50,57,57,45,52,47,40,,,
+Phi-3-mini-128k-instruct,qa5,79,66,76,72,72,73,71,64,23,,,
+ai21labs/Jamba-v0.1,qa5,83,70,64,62,58,64,63,60,50,,,
+c4ai-command-r-v01,qa5,86,82,81,78,75,79,72,70,66,,,
+Phi-3-medium-128k-instruct,qa5,77,85,84,81,82,82,81,78,69,,,
+GPT-4,qa5,96,95,92,90,93,85,82,60,51,,,
+~ Mamba (130M) fine-tune,qa5,98,99,98,99,99,99,98,99,98,,,
+Llama3-ChatQA-1.5-8B + RAG,qa5,80,77,78,77,78,77,78,76,75,75,76,67
+~ RMT (137M) fine-tune,qa5,100,100,99,99,99,94,90,89,86,69,63,44
+~ ARMT (137M) fine-tune,qa5,99.6,,,98.1,98.2,98.1,98,97.9,96.9,95.3,93.4,76.6

draw_utils.py

@@ -0,0 +1,57 @@
+import pandas as pd
+import numpy as np
+
+PAGE_MARKDOWN = """
+ <style>
+ .reportview-container {
+ margin-top: -2em;
+ }
+ #MainMenu {visibility: hidden;}
+ .stDeployButton {display:none;}
+ footer {visibility: hidden;}
+ #stDecoration {display:none;}
+ </style>
+"""
+
+PAGE_INFO = """[![Dataset on HF](https://huggingface.co/datasets/huggingface/badges/resolve/main/dataset-on-hf-lg.svg)](https://huggingface.co/datasets/booydar/babilong) | [GitHub](https://github.com/booydar/babilong) | [Paper](https://arxiv.org/abs/2406.10149) | [HF Dataset](https://huggingface.co/datasets/booydar/babilong) | [HF Dataset 1k samples per task](https://huggingface.co/datasets/RMT-team/babilong-1k-samples) |"""
+
+LENGTHS = ['0k', '1k', '2k', '4k', '8k', '16k', '32k', '64k', '128k', '512k', '1M', '2M']
+LENGTHS_32k = ['0k', '1k', '2k', '4k', '8k', '16k', '32k']
+LENGTHS_128k = ['0k', '1k', '2k', '4k', '8k', '16k', '32k', '64k', '128k']
+
+
+def load_results():
+    old_results_path = "data/leaderboard-v0_results.csv"
+    new_results_path = "babilong/babilong_results/all_results.csv"
+    old_results = pd.read_csv(old_results_path)
+    new_results = pd.read_csv(new_results_path)
+
+    res = pd.concat([old_results, new_results])
+    res.replace(-1, np.nan, inplace=True)
+    res['avg(32k)'] = res[LENGTHS_32k].mean(axis=1)
+    res['avg(128k)'] = res[LENGTHS_128k].mean(axis=1)
+    res.sort_values(['avg(128k)'], ascending=False, inplace=True)
+
+    return res
+
+
+def style_dataframe(df):
+    """
+    Style a pandas DataFrame with a color gradient.
+    """
+    styled_df = df.copy()
+    numeric_columns = styled_df.columns[1:]
+
+    def color_scale(val):
+        if pd.isna(val):
+            return 'background-color: white; color: white'
+        min_val = 0
+        max_val = 100
+        normalized = (val - min_val) / (max_val - min_val) if max_val > min_val else 0.5
+        r = int(255 * (1 - normalized) + 144 * normalized)
+        g = int(204 * (1 - normalized) + 238 * normalized)
+        b = int(204 * (1 - normalized) + 180 * normalized)
+        return f'background-color: rgb({r}, {g}, {b})'
+
+    styled = styled_df.style.map(color_scale, subset=numeric_columns)
+    return styled

requirements.txt

@@ -0,0 +1,2 @@
+streamlit
+pandas

setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Clone repositories
+git clone https://github.com/booydar/babilong -b feat/babilong_evals_hf
+git clone https://huggingface.co/datasets/RMT-team/babilong_evals babilong/babilong_evals_new
+
+# Run the evaluation script
+cd babilong
+python -m babilong.collect_results --model_name all --save_path ./babilong_results --evals_path ./babilong_evals_new