| import { Benchmark } from "./types"; | |
| export const qwenBenchmarks: Benchmark[] = [ | |
| { | |
| model: "Qwen3-235B-A22B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 87.81, | |
| //mmluredux: 87.40, | |
| ////"mmlu-pro": 68.18, | |
| ////supergpqa: 44.06, | |
| ////bbh: 88.87, | |
| gpqa: 47.47, | |
| ////gsm8k: 94.39, | |
| ////math: 71.84, | |
| ////evalplus: 77.60, | |
| //multiple: 65.94, | |
| mbpp: 81.40, | |
| //cruxo: 79.00, | |
| ////mgsm: 83.53, | |
| mmmlu: 86.70, | |
| ////include: 73.46, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-32B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 83.61, | |
| //"mmlu-redux": 83.41, | |
| //"mmlu-pro": 65.54, | |
| //supergpqa: 39.78, | |
| //bbh: 87.38, | |
| gpqa: 49.49, | |
| //gsm8k: 93.40, | |
| //math: 61.62, | |
| //evalplus: 72.05, | |
| //"multipl-e":: 67.06, | |
| mbpp: 78.20, | |
| // "crux-o":: 72.50, | |
| //mgsm: 83.06, | |
| mmmlu: 83.83, | |
| //include: 67.87, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-14B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 81.05, | |
| //"mmlu-redux": 79.88, | |
| //"mmlu-pro": 61.03, | |
| //supergpqa: 34.27, | |
| //bbh: 81.07, | |
| gpqa: 39.90, | |
| //gsm8k: 92.49, | |
| //math: 62.02, | |
| //evalplus: 72.23, | |
| //"multipl-e":: 61.69, | |
| mbpp: 73.40, | |
| // "crux-o":: 68.60, | |
| //mgsm: 79.20, | |
| mmmlu: 81.46, | |
| //include: 64.55, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-30B-A3B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 81.38, | |
| //"mmlu-redux": 81.17, | |
| //"mmlu-pro": 61.49, | |
| //supergpqa: 35.72, | |
| //bbh: 81.54, | |
| gpqa: 43.94, | |
| //gsm8k: 91.81, | |
| //math: 59.04, | |
| //evalplus: 71.45, | |
| //"multipl-e":: 66.53, | |
| mbpp: 74.40, | |
| // "crux-o":: 67.20, | |
| //mgsm: 79.11, | |
| mmmlu: 81.46, | |
| //include: 67.00, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-8B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 76.89, | |
| //"mmlu-redux": 76.17, | |
| //"mmlu-pro": 56.73, | |
| //supergpqa: 31.64, | |
| //bbh: 78.40, | |
| gpqa: 44.44, | |
| //gsm8k: 89.84, | |
| //math: 60.80, | |
| //evalplus: 67.65, | |
| //"multipl-e":: 58.75, | |
| mbpp: 69.80, | |
| // "crux-o":: 62.00, | |
| //mgsm: 76.02, | |
| mmmlu: 75.72, | |
| //include: 59.40, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-4B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 72.99, | |
| //"mmlu-redux": 72.79, | |
| //"mmlu-pro": 50.58, | |
| //supergpqa: 28.43, | |
| //bbh: 72.59, | |
| gpqa: 36.87, | |
| //gsm8k: 87.79, | |
| //math: 54.10, | |
| //evalplus: 63.53, | |
| //"multipl-e":: 53.13, | |
| mbpp: 67.00, | |
| // "crux-o":: 55.00, | |
| //mgsm: 67.74, | |
| mmmlu: 71.42, | |
| //include: 56.29, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-1.7B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 62.63, | |
| //"mmlu-redux": 61.66, | |
| //"mmlu-pro": 36.76, | |
| //supergpqa: 20.92, | |
| //bbh: 54.47, | |
| gpqa: 28.28, | |
| //gsm8k: 75.44, | |
| //math: 43.50, | |
| //evalplus: 52.70, | |
| //"multipl-e":: 42.71, | |
| mbpp: 55.40, | |
| // "crux-o":: 36.40, | |
| //mgsm: 50.71, | |
| mmmlu: 63.27, | |
| //include: 45.57, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-0.6B (Base Model)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| mmlu: 52.81, | |
| //"mmlu-redux": 51.26, | |
| //"mmlu-pro": 24.74, | |
| //supergpqa: 15.03, | |
| //bbh: 41.47, | |
| gpqa: 26.77, | |
| //gsm8k: 59.59, | |
| //math: 32.44, | |
| //evalplus: 36.23, | |
| //"multipl-e":: 24.58, | |
| mbpp: 36.60, | |
| // "crux-o":: 27.00, | |
| //mgsm: 30.99, | |
| mmmlu: 50.16, | |
| //include: 34.26, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-235B-A22B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 92.7, | |
| gpqa_diamond: 71.1, | |
| //"c-eval": 89.6, | |
| //"livebench-2024-11-25": 77.1, | |
| //"ifeval-strict-prompt": 83.4, | |
| //"arena-hard": 95.6, | |
| //"alignbench-v1.1": 8.94, | |
| //"creative-writing-v3": 84.6, | |
| //writingbench: 8.03, | |
| //"math-500": 98.0, | |
| aime_24: 85.7, | |
| aime_2025: 81.5, | |
| //zebralogic: 80.3, | |
| //autologi: 89.0, | |
| //"bfcl-v3": 70.8, | |
| //"livecodebench-v5": 70.7, | |
| //"codeforces-rating": 2056, | |
| //"codeforces-percentile": 98.2, | |
| //"multi-if": 71.9, | |
| //include: 78.7, | |
| // "mmmlu-14-languages": 84.3, | |
| //"mt-aime2024": 80.8, | |
| //poly//math: 54.7, | |
| //mlogiqa: 77.1, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-235B-A22B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 89.2, | |
| gpqa_diamond: 62.9, | |
| //"c-eval": 86.1, | |
| //"livebench-2024-11-25": 62.5, | |
| //"ifeval-strict-prompt": 83.2, | |
| //"arena-hard": 96.1, | |
| //"alignbench-v1.1": 8.91, | |
| //"creative-writing-v3": 80.4, | |
| //writingbench: 7.70, | |
| //"math-500": 91.2, | |
| aime_24: 40.1, | |
| aime_2025: 24.7, | |
| //zebralogic: 37.7, | |
| //autologi: 83.3, | |
| //"bfcl-v3": 68.0, | |
| //"livecodebench-v5": 35.3, | |
| //"codeforces-rating": 1387, | |
| //"codeforces-percentile": 75.7, | |
| //"multi-if": 70.2, | |
| //include: 75.6, | |
| // "mmmlu-14-languages": 79.8, | |
| //"mt-aime2024": 32.4, | |
| //poly//math: 27.0, | |
| //mlogiqa: 67.6, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-32B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 90.9, | |
| gpqa_diamond: 68.4, | |
| //"c-eval": 87.3, | |
| //"livebench-2024-11-25": 74.9, | |
| //"ifeval-strict-prompt": 85.0, | |
| //"arena-hard": 93.8, | |
| //"alignbench-v1.1": 8.72, | |
| //"creative-writing-v3": 81.0, | |
| //writingbench: 7.90, | |
| //"math-500": 97.2, | |
| aime_24: 81.4, | |
| aime_2025: 72.9, | |
| //zebralogic: 88.8, | |
| //autologi: 87.3, | |
| //"bfcl-v3": 70.3, | |
| //"livecodebench-v5": 65.7, | |
| //"codeforces-rating": 1977, | |
| //"codeforces-percentile": 97.7, | |
| //"multi-if": 73.0, | |
| //include: 73.7, | |
| // "mmmlu-14-languages": 80.6, | |
| //"mt-aime2024": 75.0, | |
| //poly//math: 47.4, | |
| //mlogiqa: 76.3, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-32B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 85.7, | |
| gpqa_diamond: 54.6, | |
| //"c-eval": 83.3, | |
| //"livebench-2024-11-25": 59.8, | |
| //"ifeval-strict-prompt": 83.2, | |
| //"arena-hard": 92.8, | |
| //"alignbench-v1.1": 8.58, | |
| //"creative-writing-v3": 78.3, | |
| //writingbench: 7.54, | |
| //"math-500": 88.6, | |
| aime_24: 31.0, | |
| aime_2025: 20.2, | |
| //zebralogic: 29.2, | |
| //autologi: 78.5, | |
| //"bfcl-v3": 63.0, | |
| //"livecodebench-v5": 31.3, | |
| //"codeforces-rating": 1353, | |
| //"codeforces-percentile": 71.0, | |
| //"multi-if": 70.7, | |
| //include: 70.9, | |
| // "mmmlu-14-languages": 76.5, | |
| //"mt-aime2024": 24.1, | |
| //poly//math: 22.5, | |
| //mlogiqa: 62.9, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-14B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 88.6, | |
| gpqa_diamond: 64.0, | |
| //"c-eval": 86.2, | |
| //"livebench-2024-11-25": 71.3, | |
| //"ifeval-strict-prompt": 85.4, | |
| //"arena-hard": 91.7, | |
| //"alignbench-v1.1": 8.56, | |
| //"creative-writing-v3": 80.3, | |
| //writingbench: 7.80, | |
| //"math-500": 96.8, | |
| aime_24: 79.3, | |
| aime_2025: 70.4, | |
| //zebralogic: 88.5, | |
| //autologi: 89.2, | |
| //"bfcl-v3": 70.4, | |
| //"livecodebench-v5": 63.5, | |
| //"codeforces-rating": 1766, | |
| //"codeforces-percentile": 95.3, | |
| //"multi-if": 74.8, | |
| //include: 71.7, | |
| // "mmmlu-14-languages": 77.9, | |
| //"mt-aime2024": 73.3, | |
| //poly//math: 45.8, | |
| //mlogiqa: 71.1, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-30B-A3B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 89.5, | |
| gpqa_diamond: 65.8, | |
| //"c-eval": 86.6, | |
| //"livebench-2024-11-25": 74.3, | |
| //"ifeval-strict-prompt": 86.5, | |
| //"arena-hard": 91.0, | |
| //"alignbench-v1.1": 8.70, | |
| //"creative-writing-v3": 79.1, | |
| //writingbench: 7.70, | |
| //"math-500": 98.0, | |
| aime_24: 80.4, | |
| aime_2025: 70.9, | |
| //zebralogic: 89.5, | |
| //autologi: 88.7, | |
| //"bfcl-v3": 69.1, | |
| //"livecodebench-v5": 62.6, | |
| //"codeforces-rating": 1974, | |
| //"codeforces-percentile": 97.7, | |
| //"multi-if": 72.2, | |
| //include: 71.9, | |
| // "mmmlu-14-languages": 78.4, | |
| //"mt-aime2024": 73.9, | |
| //poly//math: 46.1, | |
| //mlogiqa: 70.1, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-14B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 82.0, | |
| gpqa_diamond: 54.8, | |
| //"c-eval": 81.0, | |
| //"livebench-2024-11-25": 59.6, | |
| //"ifeval-strict-prompt": 84.8, | |
| //"arena-hard": 86.3, | |
| //"alignbench-v1.1": 8.52, | |
| //"creative-writing-v3": 73.1, | |
| //writingbench: 7.24, | |
| //"math-500": 90.0, | |
| aime_24: 31.7, | |
| aime_2025: 23.3, | |
| //zebralogic: 33.0, | |
| //autologi: 82.0, | |
| //"bfcl-v3": 61.5, | |
| //"livecodebench-v5": 29.0, | |
| //"codeforces-rating": 1200, | |
| //"codeforces-percentile": 58.6, | |
| //"multi-if": 72.9, | |
| //include: 67.8, | |
| // "mmmlu-14-languages": 72.6, | |
| //"mt-aime2024": 23.2, | |
| //poly//math: 22.0, | |
| //mlogiqa: 58.9, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-30B-A3B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 84.1, | |
| gpqa_diamond: 54.8, | |
| //"c-eval": 82.9, | |
| //"livebench-2024-11-25": 59.4, | |
| //"ifeval-strict-prompt": 83.7, | |
| //"arena-hard": 88.0, | |
| //"alignbench-v1.1": 8.55, | |
| //"creative-writing-v3": 68.1, | |
| //writingbench: 7.22, | |
| //"math-500": 89.8, | |
| aime_24: 32.8, | |
| aime_2025: 21.6, | |
| //zebralogic: 33.2, | |
| //autologi: 81.5, | |
| //"bfcl-v3": 58.6, | |
| //"livecodebench-v5": 29.8, | |
| //"codeforces-rating": 1267, | |
| //"codeforces-percentile": 64.1, | |
| //"multi-if": 70.8, | |
| //include: 67.8, | |
| // "mmmlu-14-languages": 73.8, | |
| //"mt-aime2024": 24.6, | |
| //poly//math: 23.3, | |
| //mlogiqa: 53.3, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-4B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 83.7, | |
| gpqa_diamond: 55.9, | |
| //"c-eval": 77.5, | |
| //"livebench-2024-11-25": 63.6, | |
| //"ifeval-strict-prompt": 81.9, | |
| //"arena-hard": 76.6, | |
| //"alignbench-v1.1": 8.30, | |
| //"creative-writing-v3": 61.1, | |
| //writingbench: 7.35, | |
| //"math-500": 97.0, | |
| aime_24: 73.8, | |
| aime_2025: 65.6, | |
| //zebralogic: 81.0, | |
| //autologi: 87.9, | |
| //"bfcl-v3": 65.9, | |
| //"livecodebench-v5": 54.2, | |
| //"codeforces-rating": 1671, | |
| //"codeforces-percentile": 92.8, | |
| //"multi-if": 66.3, | |
| //include: 61.8, | |
| // "mmmlu-14-languages": 69.8, | |
| //"mt-aime2024": 60.7, | |
| //poly//math: 40.0, | |
| //mlogiqa: 65.9, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-8B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 87.5, | |
| gpqa_diamond: 62.0, | |
| //"c-eval": 83.4, | |
| //"livebench-2024-11-25": 67.1, | |
| //"ifeval-strict-prompt": 85.0, | |
| //"arena-hard": 85.8, | |
| //"alignbench-v1.1": 8.46, | |
| //"creative-writing-v3": 75.0, | |
| //writingbench: 7.59, | |
| //"math-500": 97.4, | |
| aime_24: 76.0, | |
| aime_2025: 67.3, | |
| //zebralogic: 84.8, | |
| //autologi: 89.1, | |
| //"bfcl-v3": 68.1, | |
| //"livecodebench-v5": 57.5, | |
| //"codeforces-rating": 1785, | |
| //"codeforces-percentile": 95.6, | |
| //"multi-if": 71.2, | |
| //include: 67.8, | |
| // "mmmlu-14-languages": 74.4, | |
| //"mt-aime2024": 65.4, | |
| //poly//math: 42.7, | |
| //mlogiqa: 69.0, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-4B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 77.3, | |
| gpqa_diamond: 41.7, | |
| //"c-eval": 72.2, | |
| //"livebench-2024-11-25": 48.4, | |
| //"ifeval-strict-prompt": 81.2, | |
| //"arena-hard": 66.2, | |
| //"alignbench-v1.1": 8.10, | |
| //"creative-writing-v3": 53.6, | |
| //writingbench: 6.85, | |
| //"math-500": 84.8, | |
| aime_24: 25.0, | |
| aime_2025: 19.1, | |
| //zebralogic: 35.2, | |
| //autologi: 76.3, | |
| //"bfcl-v3": 57.6, | |
| //"livecodebench-v5": 21.3, | |
| //"codeforces-rating": 842, | |
| //"codeforces-percentile": 33.7, | |
| //"multi-if": 61.3, | |
| //include: 53.8, | |
| // "mmmlu-14-languages": 61.7, | |
| //"mt-aime2024": 13.9, | |
| //poly//math: 16.6, | |
| //mlogiqa: 49.9, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-8B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 79.5, | |
| gpqa_diamond: 39.3, | |
| //"c-eval": 77.9, | |
| //"livebench-2024-11-25": 53.5, | |
| //"ifeval-strict-prompt": 83.0, | |
| //"arena-hard": 79.6, | |
| //"alignbench-v1.1": 8.38, | |
| //"creative-writing-v3": 64.5, | |
| //writingbench: 7.15, | |
| //"math-500": 87.4, | |
| aime_24: 29.1, | |
| aime_2025: 20.9, | |
| //zebralogic: 26.7, | |
| //autologi: 76.5, | |
| //"bfcl-v3": 60.2, | |
| //"livecodebench-v5": 22.8, | |
| //"codeforces-rating": 1110, | |
| //"codeforces-percentile": 52.4, | |
| //"multi-if": 69.2, | |
| //include: 62.5, | |
| // "mmmlu-14-languages": 66.9, | |
| //"mt-aime2024": 16.6, | |
| //poly//math: 18.8, | |
| //mlogiqa: 51.4, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-0.6B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 55.6, | |
| gpqa_diamond: 27.9, | |
| //"c-eval": 50.4, | |
| //"livebench-2024-11-25": 30.3, | |
| //"ifeval-strict-prompt": 59.2, | |
| //"arena-hard": 8.5, | |
| //"alignbench-v1.1": 6.10, | |
| //"creative-writing-v3": 30.6, | |
| //writingbench: 5.61, | |
| //"math-500": 77.6, | |
| aime_24: 10.7, | |
| aime_2025: 15.1, | |
| //zebralogic: 30.3, | |
| //autologi: 61.6, | |
| //"bfcl-v3": 46.4, | |
| //"livecodebench-v5": 12.3, | |
| //"multi-if": 36.1, | |
| //include: 35.9, | |
| // "mmmlu-14-languages": 43.1, | |
| //"mt-aime2024": 7.8, | |
| //poly//math: 11.4, | |
| //mlogiqa: 40.9, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-1.7B (Thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 73.9, | |
| gpqa_diamond: 40.1, | |
| //"c-eval": 68.1, | |
| //"livebench-2024-11-25": 51.1, | |
| //"ifeval-strict-prompt": 72.5, | |
| //"arena-hard": 43.1, | |
| //"alignbench-v1.1": 7.60, | |
| //"creative-writing-v3": 48.0, | |
| //writingbench: 7.02, | |
| //"math-500": 93.4, | |
| aime_24: 48.3, | |
| aime_2025: 36.8, | |
| //zebralogic: 63.2, | |
| //autologi: 83.2, | |
| //"bfcl-v3": 56.6, | |
| //"livecodebench-v5": 33.2, | |
| //"multi-if": 51.2, | |
| //include: 51.8, | |
| // "mmmlu-14-languages": 59.1, | |
| //"mt-aime2024": 36.1, | |
| //poly//math: 25.2, | |
| //mlogiqa: 56.0, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-0.6B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 44.6, | |
| gpqa_diamond: 22.9, | |
| //"c-eval": 42.6, | |
| //"livebench-2024-11-25": 21.8, | |
| //"ifeval-strict-prompt": 54.5, | |
| //"arena-hard": 6.5, | |
| //"alignbench-v1.1": 5.60, | |
| //"creative-writing-v3": 28.4, | |
| //writingbench: 5.13, | |
| //"math-500": 55.2, | |
| aime_24: 3.4, | |
| aime_2025: 2.6, | |
| //zebralogic: 4.2, | |
| //autologi: 37.4, | |
| //"bfcl-v3": 44.1, | |
| //"livecodebench-v5": 3.6, | |
| //"multi-if": 33.3, | |
| //include: 34.4, | |
| // "mmmlu-14-languages": 37.1, | |
| //"mt-aime2024": 1.5, | |
| //poly//math: 4.6, | |
| //mlogiqa: 37.3, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| }, | |
| { | |
| model: "Qwen3-1.7B (Non-thinking Mode)", | |
| provider: "Qwen", | |
| inputPrice: 0, | |
| outputPrice: 0, | |
| benchmark: { | |
| //"mmlu-redux": 64.4, | |
| gpqa_diamond: 28.6, | |
| //"c-eval": 61.0, | |
| //"livebench-2024-11-25": 35.6, | |
| //"ifeval-strict-prompt": 68.2, | |
| //"arena-hard": 36.9, | |
| //"alignbench-v1.1": 7.20, | |
| //"creative-writing-v3": 43.6, | |
| //writingbench: 6.54, | |
| //"math-500": 73.0, | |
| aime_24: 13.4, | |
| aime_2025: 9.8, | |
| //zebralogic: 12.8, | |
| //autologi: 59.8, | |
| //"bfcl-v3": 52.2, | |
| //"livecodebench-v5": 11.6, | |
| //"multi-if": 44.7, | |
| //include: 42.6, | |
| // "mmmlu-14-languages": 48.3, | |
| //"mt-aime2024": 4.9, | |
| //poly//math: 10.3, | |
| //mlogiqa: 41.1, | |
| }, | |
| source: "https://arxiv.org/pdf/2505.09388", | |
| },]; | |