added more info

app.py

@@ -34,7 +34,7 @@ def create_app():
                 df, CATEGORIES, METHODOLOGY, HEADER_CONTENT, CARDS
             )
 
-            mc_info, mc_plot = create_model_comparison_tab(df, HEADER_CONTENT, CARDS)
+            mc_info, mc_plot = create_model_comparison_tab(df, HEADER_CONTENT)
 
             # exp_outputs = create_exploration_tab(
             #     df, MODELS, DATASETS, SCORES, HEADER_CONTENT

data_loader.py

@@ -538,8 +538,8 @@ HEADER_CONTENT = (
 <div class="header-wrapper">
     <div class="header-content">
         <div class="title-section">
-            <div class="subtitle-white">Welcome to the</div>
-            <div class="title-gradient">Agent Leaderboard!</div>
+            
+            <div class="title-gradient">Agent Leaderboard</div>
             
             <div class="description">
                 The landscape of AI agents is evolving rapidly, with major tech CEOs predicting 2025 as a pivotal year. 
@@ -589,7 +589,7 @@ CARDS = """        <div class="metrics-grid">
             <div class="metric-card">
                 <div class="metric-number metric-purple">14</div>
                 <div class="metric-label">Evaluation Datasets</div>
-                <div class="metric-detail primary">Cross-Domain Testing</div>
+                <div class="metric-detail primary">Multi-Domain Testing</div>
                 <div class="metric-detail primary">Real-world use cases</div>
             </div>
 
@@ -708,20 +708,20 @@ METHODOLOGY = """
         font-size: 0.9rem;
     }
 
-    [Rest of the CSS remains the same]
-</style>
-<!-- Methodology Section -->
-<div class="methodology-section">
-    <h1 class="methodology-title">Methodology</h1>
+    .code-intro {
+    color: var(--text-secondary);
+    font-size: 1.1rem;
+    margin-bottom: 1.5rem;
+    line-height: 1.6;
+    }
     
-    <h2 class="methodology-subtitle">Overview</h2>
-    <p class="methodology-text">
-        We evaluate language models' ability to effectively use tools 
-        in single and multi-turn conversations. Our evaluation focuses on both basic functionality and edge 
-        cases that challenge real-world applicability.
-    </p>
+    .section-divider {
+        margin: 1rem 0;
+        border-top: 1px solid var(--border-color);
+        opacity: 0.3;
+    }
+
 
-<style>
 .key-insights thead tr {
     background: linear-gradient(90deg, #60A5FA, #818CF8);
 }
@@ -739,10 +739,235 @@ METHODOLOGY = """
     padding: 1rem;
     border-bottom: 1px solid rgba(31, 41, 55, 0.5);
 }
+
+.highlight {
+    color: var(--accent-blue);
+    font-weight: 600;
+    display: inline-flex;
+    align-items: center;
+}
+
+.highlight::after {
+    content: ":";
+    margin-right: 0.5rem;  /* Adds space after the colon */
+}
+
+  @media (prefers-color-scheme: dark) {
+        :root {
+            --bg-primary: #0B0B19;
+            --bg-secondary: rgba(19, 19, 37, 0.4);
+            --text-primary: #ffffff;
+            --text-secondary: #94A3B8;
+            --border-color: rgba(31, 41, 55, 0.5);
+            --accent-blue: #60A5FA;
+            --accent-purple: #A78BFA;
+            --code-bg: #1E1E2E;
+            --code-line-highlight: rgba(96, 165, 250, 0.1);
+            --bullet-color: #60A5FA;
+            --table-header: #1a1b1e;
+            --table-border: #2d2e32;
+            --table-hover: #2d2e32;
+        }
+    }
+    
+    @media (prefers-color-scheme: light) {
+        :root {
+            --bg-primary: #ffffff;
+            --bg-secondary: rgba(243, 244, 246, 0.4);
+            --text-primary: #111827;
+            --text-secondary: #4B5563;
+            --border-color: rgba(209, 213, 219, 0.5);
+            --accent-blue: #3B82F6;
+            --accent-purple: #8B5CF6;
+            --code-bg: #F8FAFC;
+            --code-line-highlight: rgba(59, 130, 246, 0.1);
+            --bullet-color: #3B82F6;
+            --table-header: #F8FAFC;
+            --table-border: #E5E7EB;
+            --table-hover: #F3F4F6;
+        }
+    }
+
+    .methodology-content {
+        max-width: 1200px;
+        margin: 0 auto;
+        padding: 2rem;
+        color: var(--text-secondary);
+        line-height: 1.7;
+        font-size: 1rem;
+    }
+
+    .section-title {
+        font-size: 2.5rem;
+        font-weight: 700;
+        margin: 3rem 0 1.5rem;
+        color: var(--text-primary);
+        background: linear-gradient(to right, var(--accent-blue), var(--accent-purple));
+        -webkit-background-clip: text;
+        -webkit-text-fill-color: transparent;
+        letter-spacing: -0.02em;
+    }
+
+    .subsection-title {
+        font-size: 1.75rem;
+        font-weight: 600;
+        margin: 2rem 0 1rem;
+        color: var(--text-primary);
+        letter-spacing: -0.01em;
+    }
+
+    .content-block {
+        background: var(--bg-secondary);
+        border: 1px solid var(--border-color);
+        border-radius: 12px;
+        padding: 1.5rem;
+        margin: 1.5rem 0;
+    }
+
+    .methodology-list {
+    list-style: none !important;  /* Force remove default bullets */
+    padding: 0;
+    margin: 1rem 0;
+}
+
+    .methodology-list li {
+        padding-left: 2rem;
+        position: relative;
+        margin: 1rem 0;
+        color: var(--text-secondary);
+        display: flex;  /* Add flex display */
+        align-items: flex-start;  /* Align items to top */
+    }
+
+    .methodology-list li::before {
+        content: '';
+        position: absolute;
+        left: 0;
+        top: 0.75rem;
+        width: 8px;
+        height: 8px;
+        background: var(--bullet-color);
+        border-radius: 50%;
+        box-shadow: 0 0 0 2px rgba(96, 165, 250, 0.2);
+        flex-shrink: 0;  /* Prevent bullet from shrinking */
+    }
+
+    /* Additional fix for nested list items */
+    .methodology-list li > * {
+        list-style: none !important;
+        margin-left: 0;
+        padding-left: 0;
+    }
+
+    .code-block {
+        background: var(--code-bg);
+        border-radius: 12px;
+        padding: 1.5rem;
+        margin: 1.5rem 0;
+        font-family: 'SF Mono', 'Menlo', monospace;
+        font-size: 0.9rem;
+        overflow-x: auto;
+        border: 1px solid var(--border-color);
+    }
+
+    .code-block pre {
+        margin: 0;
+        padding: 0;
+    }
+
+    .highlight {
+        color: var(--accent-blue);
+        font-weight: 600;
+    }
+
+    /* Dataset Table Styling */
+    .dataset-table {
+        width: 100%;
+        border-collapse: collapse;
+        margin: 1.5rem 0;
+        background: var(--bg-secondary);
+        border-radius: 12px;
+        overflow: hidden;
+    }
+
+    .dataset-table th {
+        background: var(--table-header);
+        padding: 1rem;
+        text-align: left;
+        font-weight: 600;
+        color: var(--text-primary);
+        border-bottom: 2px solid var(--table-border);
+    }
+
+    .dataset-table td {
+        padding: 1rem;
+        border-bottom: 1px solid var(--table-border);
+        color: var(--text-secondary);
+    }
+
+    .dataset-table tbody tr:hover {
+        background: var(--table-hover);
+    }
+
+    .dataset-table td:first-child {
+        font-weight: 500;
+    }
+    
 </style>
 
+<!-- Methodology Section -->
+        <h1 class="section-title">Methodology</h1>
+    <p>Our evaluation process follows a systematic approach to ensure comprehensive and fair assessment of AI agents. We evaluate language models' ability to effectively use tools 
+        in single and multi-turn conversations. Our evaluation focuses on both basic functionality and edge 
+        cases that challenge real-world applicability.</p>
+
+    <ul class="methodology-list">
+            <li><span class="highlight">Model Selection</span>We begin by curating a diverse set of leading language models, including both proprietary and open-source implementations.</li>
+            <li><span class="highlight">Agent Configuration</span>Each model is configured as an agent using a standardized system prompt and given access to a consistent set of tools.</li>
+            <li><span class="highlight">Metric Definition</span> <a href="https://docs.galileo.ai/galileo/gen-ai-studio-products/galileo-guardrail-metrics/tool-selection-quality#tool-selection-quality">Tool Selection Quality (TSQ)</a></li>
+            <li><span class="highlight">Scoring System</span>The final performance score is calculated as an equally weighted average across all datasets.</li>
+            <li><span class="highlight">Dataset Curation</span>We strategically sampled from established benchmarking datasets. See later section for more info.</li>
+    <div class="methodology-section">
+        <div class="table-container">
+            <table class="dataset-table">
+                <thead>
+                    <tr>
+                        <th>Dataset</th>
+                        <th>Domains</th>
+                        <th>Link</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <tr>
+                        <td>BFCL</td>
+                        <td>Mathematics, Entertainment, Education, and Academic Domains</td>
+                        <td><a href="https://gorilla.cs.berkeley.edu/leaderboard.html">View Dataset</a></td>
+                    </tr>
+                    <tr>
+                        <td>τ-bench</td>
+                        <td>Retail and Airline Industry Scenarios</td>
+                        <td><a href="https://github.com/sierra-research/tau-bench">View Dataset</a></td>
+                    </tr>
+                    <tr>
+                        <td>xLAM</td>
+                        <td>Cross-domain Data Generation (21 Domains)</td>
+                        <td><a href="https://www.salesforce.com/blog/xlam-large-action-models/">View Dataset</a></td>
+                    </tr>
+                    <tr>
+                        <td>ToolACE</td>
+                        <td>API Interactions across 390 Domains</td>
+                        <td><a href="https://arxiv.org/abs/2409.00920">View Dataset</a></td>
+                    </tr>
+                </tbody>
+            </table>
+        </div>
+    </div>
+
+            
+        </ul> 
+
 <div class="methodology-section">
-    <h1 class="methodology-subtitle">Key Insights</h1>
+    <h1 class="section-title">Key Insights</h1>
     <div class="table-container">
         <table class="dataset-table key-insights">
             <thead>
@@ -754,11 +979,11 @@ METHODOLOGY = """
             <tbody>
                 <tr>
                     <td>Performance Champion</td>
-                    <td>Gemini-2.0-flash dominates with 0.935 score at just $0.075 per million tokens, excelling in both complex tasks (0.95) and safety features (0.98)</td>
+                    <td>Gemini-2.0-flash dominates with 0.938 score at a very affordable cost, excelling in both complex tasks and safety features.</td>
                 </tr>
                 <tr>
                     <td>Price-Performance Paradox</td>
-                    <td>Top 3 models span 20x price difference yet only 3% performance gap, challenging pricing assumptions</td>
+                    <td>Top 3 models span 10x price difference yet only 4% performance gap, challenging pricing assumptions</td>
                 </tr>
                 <tr>
                     <td>Open Vs Closed Source</td>
@@ -770,7 +995,7 @@ METHODOLOGY = """
                 </tr>
                 <tr>
                     <td>Tool Miss Detection</td>
-                    <td>Dataset averages of 0.59 and 0.78 reveal fundamental challenges in handling edge cases and maintaining context, even as models excel at basic tasks</td>
+                    <td>Low dataset averages of 0.60(tool_miss) and 0.73(miss_func) reveal fundamental challenges in handling edge cases and maintaining context, even as models excel at basic tasks</td>
                 </tr>
                 <tr>
                     <td>Architecture Trade-offs</td>
@@ -780,7 +1005,7 @@ METHODOLOGY = """
         </table>
     </div>
 
-    <h2 class="methodology-subtitle">Development Implications</h2>
+    <h1 class="section-title">Development Implications</h2>
     <div class="table-container">
         <table class="dataset-table key-insights">
             <thead>
@@ -818,7 +1043,92 @@ METHODOLOGY = """
         </table>
     </div>
 
-    <h2 class="methodology-subtitle">Dataset Structure</h2>
+
+    <div></div>
+    <h1 class="section-title">How Do We Measure Agent's Performance?</h1>
+    <div>
+        <p>The complexity of tool calling extends far beyond simple API invocations. We developed the Tool Selection Quality metric to assess agents' tool call performance, evaluating tool selection accuracy and effectiveness of parameter usage.</p>
+
+    <div class="section-divider"></div>
+    <h2 class="subsection-title">Scenario Recognition</h2>
+    <div class="explanation-block">
+        <p>When an agent encounters a query, it must first determine if tool usage is warranted. Information may already exist in the conversation history, making tool calls redundant. Alternatively, available tools might be insufficient or irrelevant to the task, requiring the agent to acknowledge limitations rather than force inappropriate tool usage.</p>
+    </div>
+
+    <div class="section-divider"></div>
+    <h2 class="subsection-title">Tool Selection Dynamics</h2>
+    <div class="explanation-block">
+        <p>Tool selection isn't binary—it involves both precision and recall. An agent might correctly identify one necessary tool while missing others (recall issue) or select appropriate tools alongside unnecessary ones (precision issue). While suboptimal, these scenarios represent different severity levels of selection errors.</p>
+    </div>
+
+    <div class="section-divider"></div>
+    <h2 class="subsection-title">Parameter Handling</h2>
+    <div class="explanation-block">
+        <p>Even with correct tool selection, argument handling introduces additional complexity. Agents must:</p>
+        <ul class="methodology-list">
+            <li>Provide all required parameters with correct naming</li>
+            <li>Handle optional parameters appropriately</li>
+            <li>Maintain parameter value accuracy</li>
+            <li>Format arguments according to tool specifications</li>
+        </ul>
+    </div>
+    
+    <div class="section-divider"></div>
+    <h2 class="subsection-title">Sequential Decision Making</h2>
+    <div class="explanation-block">
+        <p>Multi-step tasks require agents to:</p>
+        <ul class="methodology-list">
+            <li>Determine optimal tool calling sequence</li>
+            <li>Handle interdependencies between tool calls</li>
+            <li>Maintain context across multiple operations</li>
+            <li>Adapt to partial results or failures</li>
+        </ul>
+    </div>
+    
+<div class="section-divider"></div>
+<h2 class="subsection-title">Example code</h2>
+<p class="code-intro">Below is the code example of evaluating an LLM for a dataset.</p>
+        <div class="code-block">
+         
+            <pre>
+import promptquality as pq
+
+df = pd.read_parquet(file_path)
+
+chainpoll_tool_selection_scorer = pq.CustomizedChainPollScorer(
+                scorer_name=pq.CustomizedScorerName.tool_selection_quality,
+                model_alias=pq.Models.gpt_4o,
+            )
+
+evaluate_handler = pq.GalileoPromptCallback(
+        project_name=project_name,
+        run_name=run_name,
+        scorers=[chainpoll_tool_selection_scorer],
+    )
+
+llm = llm_handler.get_llm(model, temperature=0.0, max_tokens=4000)
+system_msg = {
+            "role": "system",
+            "content": 'Your job is to use the given tools to answer the query of human. If there is no relevant tool then reply with "I cannot answer the question with given tools". If tool is available but sufficient information is not available, then ask human to get the same. You can call as many tools as you want. Use multiple tools if needed. If the tools need to be called in a sequence then just call the first tool.',
+        }
+
+for row in df.itertuples():
+    chain = llm.bind_tools(tools)
+    outputs.append(
+            chain.invoke(
+                [system_msg, *row.conversation], 
+                config=dict(callbacks=[evaluate_handler])
+            )
+        )
+
+evaluate_handler.finish()
+
+            </pre>
+        </div>
+    </div>
+
+
+    <h1 class="section-title">Dataset Structure</h2>
     <div class="table-container">
         <table class="dataset-table">
             <thead>
@@ -832,10 +1142,10 @@ METHODOLOGY = """
             </thead>
             <tbody>
                 <tr>
-                    <td rowspan="4">Single-Turn</td>
+                    <td rowspan="5">Single-Turn</td>
                     <td>100 + 100</td>
                     <td class="category-cell">Single Function Call</td>
-                    <td class="dataset-name">xlam_single_tool_single_call</td>
+                    <td class="dataset-name">xlam_single_tool_single_call, xlam_multiple_tool_single_call</td>
                     <td class="purpose-cell">Evaluates basic ability to read documentation and make single function calls</td>
                 </tr>
                 <tr>
@@ -856,6 +1166,12 @@ METHODOLOGY = """
                     <td class="dataset-name">tau_long_context</td>
                     <td class="purpose-cell">Assesses handling of extended interactions and complex instructions</td>
                 </tr>
+                <tr>
+                    <td>100</td>
+                    <td class="category-cell">Missing Function</td>
+                    <td class="dataset-name">xlam_tool_miss</td>
+                    <td class="purpose-cell">Tests graceful handling of unavailable tools</td>
+                </tr>
                 <tr>
                     <td rowspan="5">Multi-Turn</td>
                     <td>50 + 30</td>
@@ -890,6 +1206,16 @@ METHODOLOGY = """
             </tbody>
         </table>
     </div>
+    
+    <div class="section-divider"></div>
+    <h2 class="section-title">Citation</h2>
+    <div class="bibtex-citation" style="font-family: monospace; white-space: pre; padding: 1em; background-color: rgba(128, 128, 128, 0.1); border: 1px solid rgba(128, 128, 128, 0.2); border-radius: 4px; color: currentColor;">@misc{agent-leaderboard,
+    author = {Pratik Bhavsar},
+    title = {Agent Leaderboard},
+    year = {2025},
+    publisher = {Galileo.ai},
+    howpublished = {\\url{https://huggingface.co/spaces/galileo-ai/agent-leaderboard}}
+}</div>
 
 <!-- Features Grid Section -->
 <div class="features-grid">
@@ -915,11 +1241,12 @@ METHODOLOGY = """
         </div>
         <h3 class="feature-title">360° Domain Evaluation</h3>
         <ul class="feature-list">
-            <li>Cross-domain evaluation</li>
+            <li>Multi-domain evaluation</li>
             <li>Real-world use cases</li>
             <li>Edge case evaluation</li>
         </ul>
     </div>
+    
 
     <div class="feature-card">
         <div class="feature-icon">
@@ -935,4 +1262,8 @@ METHODOLOGY = """
         </ul>
     </div>
 
+</div>
+
+
+
 """

results.csv

@@ -1,5 +1,5 @@
 Model,Model Type,Model Output Type,Vendor,Input cost per million token,Output cost per million token,Model Avg,single turn perf,multi turn perf,BFCL_v3_multi_turn_base_multi_func_call,BFCL_v3_multi_turn_composite,tau_long_context,xlam_single_tool_multiple_call,BFCL_v3_multi_turn_miss_param,xlam_multiple_tool_single_call,xlam_tool_miss,BFCL_v3_multi_turn_long_context,BFCL_v3_irrelevance,BFCL_v3_multi_turn_base_single_func_call,xlam_single_tool_single_call,xlam_multiple_tool_multiple_call,BFCL_v3_multi_turn_miss_func,toolace_single_func_call
-gemini-2.0-flash-exp,Private,Normal,Google,0.1,0.4,0.935,0.94,0.93,0.86,0.95,0.9,0.99,0.95,0.94,0.83,0.91,0.98,0.96,0.98,0.98,0.88,0.975
+gemini-2.0-flash-001,Private,Normal,Google,0.15,0.6,0.938,0.95,0.93,0.91,0.94,0.9,0.96,0.92,0.95,0.89,0.91,0.98,0.93,0.97,0.98,0.93,0.965
 gpt-4o-2024-11-20,Private,Normal,OpenAI,2.5,10,0.900,0.92,0.88,0.85,0.9,0.92,0.95,0.88,0.99,0.63,0.83,0.98,0.89,0.98,0.98,0.86,0.965
 gemini-1.5-flash,Private,Normal,Google,0.075,0.3,0.895,0.88,0.91,0.9,0.9,0.89,0.87,0.91,0.83,0.71,0.87,0.98,0.89,0.94,0.93,0.92,0.99
 gemini-1.5-pro,Private,Normal,Google,1.25,5,0.885,0.87,0.91,0.89,0.93,0.75,0.97,0.9,0.87,0.57,0.91,0.94,0.92,0.99,0.97,0.86,0.925
@@ -16,4 +16,4 @@ mistral-small-2409,Private,Normal,Mistral,0.2,0.6,0.750,0.82,0.68,0.7,0.77,0.72,
 ministral-8b-2410,Private,Normal,Mistral,0.1,0.1,0.689,0.73,0.65,0.75,0.59,0.73,0.98,0.66,0.98,0.34,0.78,0.24,0.81,0.9,0.95,0.53,0.41
 Meta-Llama-3.1-8B-Instruct-Turbo,Open source,Normal,Meta,0.2,0.2,0.678,0.71,0.64,0.77,0.49,0.44,0.96,0.66,0.98,0.25,0.73,0.48,0.76,0.93,0.96,0.51,0.575
 open-mistral-nemo-2407,Open source,Normal,Mistral,0.15,0.15,0.661,0.68,0.64,0.7,0.64,0.51,0.98,0.68,0.99,0.26,0.78,0.21,0.75,0.9,0.94,0.51,0.41
-,,,,,,,0.83,0.79,0.81,0.78,0.76,0.89,0.80,0.96,0.60,0.81,0.82,0.82,0.92,0.85,0.73,0.80
+,,,,,,,0.83,0.79,0.81,0.78,0.76,0.88,0.80,0.96,0.60,0.81,0.82,0.81,0.92,0.85,0.73,0.80

tabs/leaderboard.py

@@ -126,7 +126,7 @@ def filter_leaderboard(df, model_type, category, sort_by):
                     <th>Type</th>
                     <th>Vendor</th>
                     <th>Cost (I/O)</th>
-                    <th>Category Score</th>
+                    <th>Avg Category Score (TSQ)</th>
                 </tr>
             </thead>
             <tbody>

tabs/model_comparison.py

@@ -47,7 +47,7 @@ def compare_models(df, model_names=None):
     return info_html, radar_chart
 
 
-def create_model_comparison_tab(df, HEADER_CONTENT, CARDS):
+def create_model_comparison_tab(df, HEADER_CONTENT):
     with gr.Tab("Model Comparison"):
         gr.HTML(HEADER_CONTENT)
         with gr.Column():

visualization.py

@@ -39,7 +39,12 @@ def get_performance_chart(df, category_name="Overall"):
             fontweight="bold",
             color=colors["text"],
         )
-        ax.set_xlabel("Average Score", fontsize=14, labelpad=10, color=colors["text"])
+        ax.set_xlabel(
+            "Average Score (Tool Selection Quality)",
+            fontsize=14,
+            labelpad=10,
+            color=colors["text"],
+        )
         ax.set_xlim(0.0, 1.0)
 
         ax.set_yticks(np.arange(len(df_sorted)))
@@ -222,7 +227,7 @@ def get_performance_cost_chart(df, category_name="Overall"):
 
     ax.set_title(
         f"Performance vs. Cost - {category_name}",
-        fontsize=12,
+        fontsize=14,
         pad=15,
         fontweight="bold",
         color=colors["text"],