x

src/lib/benchmarks/ index.ts

@@ -1,11 +1,13 @@
 import { Benchmark } from "./types";
 import { xaiBenchmarks } from "./xai";
 import { googleBenchmarks } from "./google";
+import { anthropicBenchmarks } from "./anthropic";
 // import other sources here as you add them
 // import { openaiBenchmarks } from "./openai";
 
 export const benchmarkData: Benchmark[] = [
   ...xaiBenchmarks,
   ...googleBenchmarks,
+  ...anthropicBenchmarks
   // ...openaiBenchmarks,
 ];

src/lib/benchmarks/anthropic.ts

@@ -0,0 +1,84 @@
+import { Benchmark } from "./types";
+
+export const anthropicBenchmarks: Benchmark[] = [
+    {
+        model: "Claude Opus 4",
+        provider: "Anthropic",
+        inputPrice: 15.0,
+        outputPrice: 75.0,
+        source: "https://www.anthropic.com/news/claude-4",
+        benchmark: {
+            swe_bench_verified: 72.5,
+            //terminal_bench: 43.2,
+            gpqa_diamond: 79.6,
+            aime_2025: 75.5,
+            //mmmlu: 88.8,
+            mmmu: 76.5,
+            // tau_bench_retail: 81.4,
+            //tau_bench_airline: 59.6,
+        },
+    },
+    {
+        model: "Claude Sonnet 4",
+        provider: "Anthropic",
+        inputPrice: 3.0,
+        outputPrice: 15.0,
+        source: "https://www.anthropic.com/news/claude-4",
+        benchmark: {
+            swe_bench_verified: 72.7,
+            //terminal_bench: 35.5,
+            gpqa_diamond: 75.4,
+            aime_2025: 70.5,
+            // mmmlu: 86.5,
+            mmmu: 74.4,
+            // tau_bench_retail: 80.5,
+            // tau_bench_airline: 60.0,
+        },
+    },
+    {
+        model: "Claude 3.7 Sonnet (Extended Thinking 64K)",
+        provider: "Anthropic",
+        inputPrice: 3.0,
+        outputPrice: 15.0,
+        source: "https://www.anthropic.com/news/claude-3-7-sonnet",
+        benchmark: {
+            gpqa_diamond: 78.2,
+           
+            //  tau_bench_retail: 81.2,
+            //  tau_bench_airline: 58.4,
+            //  mmmlu: 86.1,
+            mmmu: 75.0,
+            aime_24: 61.3, 
+        },
+    },
+    {
+        model: "Claude 3.7 Sonnet (No Extended Thinking)",
+        provider: "Anthropic",
+        inputPrice: 3.0,
+        outputPrice: 15.0,
+        source: "https://www.anthropic.com/news/claude-3-7-sonnet",
+        benchmark: {
+            gpqa_diamond: 68.0,
+            swe_bench_verified: 62.3, 
+            //  mmmlu: 83.2,
+            mmmu: 71.8,
+            aime_24: 51.7, // using average of 23.3 & 80.0
+        },
+    },
+    {
+        model: "Claude 3.5 Sonnet (New)",
+        provider: "Anthropic",
+        inputPrice: 3.0,
+        outputPrice: 15.0,
+        source: "https://www.anthropic.com/news/claude-3-7-sonnet",
+        benchmark: {
+            gpqa_diamond: 65.0,
+            swe_bench_verified: 49.0,
+            // tau_bench_retail: 71.5,
+            //  tau_bench_airline: 48.8,
+            //  mmmlu: 82.1,
+            mmmu: 70.4,
+            aime_24: 16.0, // average of 16.0 & 65.4
+        },
+    },
+];

src/lib/benchmarks/google.ts

@@ -43,6 +43,52 @@ export const googleBenchmarks: Benchmark[] = [
             mmmu: 82.0,
         },
     },
+
+    {
+    model: "Gemini 2.5 Pro Experimental (03-25)",
+    provider: "Google",
+    inputPrice: 1.25,
+    outputPrice: 10.0,
+    source: "https://blog.google/products/gemini/gemini-2-5-pro-updates/",
+    benchmark: {
+        livecodebench_v6: 70.4,
+        aider_polyglot: 74.0,
+        swe_bench_verified: 63.8,
+        gpqa_diamond: 84.0,
+        aime_2025: 86.7,
+        humanitys_last_exam: 18.8,
+        simpleqa: 52.9,
+        global_mmlu_lite: 89.8,
+        mrcr_v2_avg_128k: 94.5,
+        mrcr_v2_pointwise_1m: 83.1,
+        mmmu: 81.7,
+        // vibe_eval: 69.4,
+        // video_mme: not reported
+    },
+},
+{
+    model: "Gemini 2.5 Pro Preview (05-06)",
+    provider: "Google",
+    inputPrice: 1.25,
+    outputPrice: 10.0,
+    source: "https://blog.google/products/gemini/gemini-2-5-pro-updates/",
+    benchmark: {
+        livecodebench_v6: 75.6,
+        aider_polyglot: 76.5,
+        swe_bench_verified: 63.2,
+        gpqa_diamond: 83.0,
+        aime_2025: 83.0,
+        humanitys_last_exam: 17.8,
+        simpleqa: 50.8,
+        global_mmlu_lite: 88.6,
+        mrcr_v2_avg_128k: 93.0,
+        mrcr_v2_pointwise_1m: 82.9,
+        mmmu: 79.6,
+        // vibe_eval: 65.6,
+        // video_mme: 84.8,
+    },
+},
+
     {
         model: "Gemini 2.5 Flash (Thinking-enabled, default)",
         provider: "Google",