{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-10-11", "retrieved_at": "2026-10-11T01:46:09.977Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2173, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9483723333333332, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20306945454545455, "avg_duration_per_task_ms": 134835.03535353535, "avg_output_tokens_per_task": 16798.409090909092, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021931015151515153, "avg_duration_per_task_ms": 68600.9191919192, "avg_output_tokens_per_task": 1835.4242424242425, "last_run_timestamp": "2026-10-03T01:13:05.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.947009, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.07090916923076923, "avg_duration_per_task_ms": 142045.61538461538, "avg_output_tokens_per_task": 18762.907692307694, "last_run_timestamp": "2026-10-01T08:30:23.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11987038888888889, "avg_duration_per_task_ms": 35572.169191919194, "avg_output_tokens_per_task": 2020.9419191919192, "last_run_timestamp": "2026-10-01T08:24:02.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.015926459595959598, "avg_duration_per_task_ms": 24665.858585858587, "avg_output_tokens_per_task": 2219.030303030303, "last_run_timestamp": "2026-09-28T18:11:34.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.9393935, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.29793838636363634, "avg_duration_per_task_ms": 106861.7196969697, "avg_output_tokens_per_task": 11692.656565656565, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.937943329787234, "accuracy_std_dev": 0.011200313606720516, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.023761370567375886, "avg_duration_per_task_ms": 52418.78723404255, "avg_output_tokens_per_task": 7636.52304964539, "last_run_timestamp": "2026-10-01T16:16:37.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9371493333333333, "accuracy_std_dev": 0.004199386807089248, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.30925494444444446, "avg_duration_per_task_ms": 152328.37037037036, "avg_output_tokens_per_task": 10179.092592592593, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.9337823333333333, "accuracy_std_dev": 0.008398773614178557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3842955404040404, "avg_duration_per_task_ms": 82918.1936026936, "avg_output_tokens_per_task": 11996.59090909091, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9332206666666667, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.24476150168350166, "avg_duration_per_task_ms": 101605.01851851853, "avg_output_tokens_per_task": 11508.424242424242, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9326596250000001, "accuracy_std_dev": 0.00862529761714779, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.32610833964646463, "avg_duration_per_task_ms": 161836.85416666666, "avg_output_tokens_per_task": 10112.390782828283, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9325395714285714, "accuracy_std_dev": 0.004401689845956867, "run_count": 4, "total_questions": 756, "avg_cost_per_task_usd": 0.06429072222222224, "avg_duration_per_task_ms": 63071.10317460317, "avg_output_tokens_per_task": 11529.871693121693, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9309763333333333, "accuracy_std_dev": 0.00880159775899302, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.026392959034792367, "avg_duration_per_task_ms": 18809.703142536477, "avg_output_tokens_per_task": 2248.6481481481483, "last_run_timestamp": "2026-10-10T08:25:06.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.930415, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02488252693602694, "avg_duration_per_task_ms": 19612.508417508416, "avg_output_tokens_per_task": 2247.9511784511783, "last_run_timestamp": "2026-10-10T08:14:45.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.930415, "accuracy_std_dev": 0.004827415699246697, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06216489141414142, "avg_duration_per_task_ms": 64761.851851851854, "avg_output_tokens_per_task": 11730.429292929293, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9297306163895488, "accuracy_std_dev": 0.01120962874243434, "run_count": 9, "total_questions": 1684, "avg_cost_per_task_usd": 0.1273893414489311, "avg_duration_per_task_ms": 147661.65023752968, "avg_output_tokens_per_task": 8445.026128266034, "last_run_timestamp": "2026-10-10T08:22:26.915Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.9277397342419081, "accuracy_std_dev": 0.007149525570895533, "run_count": 6, "total_questions": 1174, "avg_cost_per_task_usd": 0.1081421950596252, "avg_duration_per_task_ms": 192835.26320272574, "avg_output_tokens_per_task": 9335.29471890971, "last_run_timestamp": "2026-10-10T08:22:51.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9275660041194645, "accuracy_std_dev": 0.01457727458237649, "run_count": 6, "total_questions": 971, "avg_cost_per_task_usd": 0.10551570648815653, "avg_duration_per_task_ms": 239156.22245108135, "avg_output_tokens_per_task": 8138.333676622039, "last_run_timestamp": "2026-10-10T08:25:06.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.927048, "accuracy_std_dev": 0.0020998715833752102, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025530393939393943, "avg_duration_per_task_ms": 19102.368686868685, "avg_output_tokens_per_task": 2313.7003367003367, "last_run_timestamp": "2026-10-10T08:14:48.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9269767035321367, "accuracy_std_dev": 0.007983506267308428, "run_count": 28, "total_questions": 5181, "avg_cost_per_task_usd": 0.16678585485427527, "avg_duration_per_task_ms": 152403.25902335456, "avg_output_tokens_per_task": 9054.98861223702, "last_run_timestamp": "2026-10-10T08:23:04.185Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.004418715424192887, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05958464141414141, "avg_duration_per_task_ms": 58983.43771043771, "avg_output_tokens_per_task": 11740.584175084176, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.926487, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02518170033670034, "avg_duration_per_task_ms": 20924.996632996634, "avg_output_tokens_per_task": 2280.7272727272725, "last_run_timestamp": "2026-10-10T08:45:10.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.925926, "accuracy_std_dev": 0.005498287975967322, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1441699393939394, "avg_duration_per_task_ms": 61658.986531986535, "avg_output_tokens_per_task": 15894.8367003367, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9259256666666668, "accuracy_std_dev": 0.0036366213562713575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1413088872053872, "avg_duration_per_task_ms": 82468.60606060606, "avg_output_tokens_per_task": 15584.752525252525, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9255440309278352, "accuracy_std_dev": 0.0047577793839843644, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.3407643092783505, "avg_duration_per_task_ms": 132309.46391752578, "avg_output_tokens_per_task": 10197.671821305841, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9248735, "accuracy_std_dev": 0.00862277742377712, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10364464204545455, "avg_duration_per_task_ms": 55123.33648989899, "avg_output_tokens_per_task": 3517.8642676767677, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9242421666666667, "accuracy_std_dev": 0.0048599613653014136, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.14878865656565657, "avg_duration_per_task_ms": 85040.50420875421, "avg_output_tokens_per_task": 15641.649831649831, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto", "model_slug": "unbiased/pareto", "model_name": "Pareto", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03377419696969697, "avg_duration_per_task_ms": 114763.54545454546, "avg_output_tokens_per_task": 4358.363636363636, "last_run_timestamp": "2026-09-29T03:11:33.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.9242419999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025158247474747476, "avg_duration_per_task_ms": 20743.68181818182, "avg_output_tokens_per_task": 2280.7853535353534, "last_run_timestamp": "2026-10-10T08:14:39.188Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.008625346413333228, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05533751388888889, "avg_duration_per_task_ms": 61705.84722222222, "avg_output_tokens_per_task": 3564.286616161616, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9228396666666667, "accuracy_std_dev": 0.0024135722257452555, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.12293840404040404, "avg_duration_per_task_ms": 199289.85858585857, "avg_output_tokens_per_task": 9078.558922558923, "last_run_timestamp": "2026-10-10T08:23:18.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9227054347826086, "accuracy_std_dev": 0.0037342422507786015, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.09927257048748353, "avg_duration_per_task_ms": 52555.73781291173, "avg_output_tokens_per_task": 3495.6416337285905, "last_run_timestamp": "2026-10-01T19:12:57.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.9225589999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15024601262626264, "avg_duration_per_task_ms": 83108.62878787878, "avg_output_tokens_per_task": 9763.2601010101, "last_run_timestamp": "2026-10-10T08:24:29.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9211143939126598, "accuracy_std_dev": 0.007124160458822897, "run_count": 12, "total_questions": 2267, "avg_cost_per_task_usd": 0.15088417026907808, "avg_duration_per_task_ms": 133150.59550066167, "avg_output_tokens_per_task": 9907.07454786061, "last_run_timestamp": "2026-10-10T08:23:05.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06043574579124578, "avg_duration_per_task_ms": 62971.38047138047, "avg_output_tokens_per_task": 3586.8703703703704, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.9206347142857143, "accuracy_std_dev": 0.006819412411099165, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.17132878354978356, "avg_duration_per_task_ms": 313924.4264069264, "avg_output_tokens_per_task": 9747.513708513708, "last_run_timestamp": "2026-10-10T08:21:47.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9205385999999999, "accuracy_std_dev": 0.008066750289924707, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1561665070707071, "avg_duration_per_task_ms": 221042.0797979798, "avg_output_tokens_per_task": 12387.791919191919, "last_run_timestamp": "2026-10-10T08:21:47.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9201543251770765, "accuracy_std_dev": 0.015248606655609568, "run_count": 8, "total_questions": 1553, "avg_cost_per_task_usd": 0.25049747198969735, "avg_duration_per_task_ms": 407837.91500321956, "avg_output_tokens_per_task": 36515.1790083709, "last_run_timestamp": "2026-10-10T08:27:33.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9197903617021278, "accuracy_std_dev": 0.011919658050883864, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.13309346149949342, "avg_duration_per_task_ms": 301635.15045592707, "avg_output_tokens_per_task": 10335.872340425532, "last_run_timestamp": "2026-10-10T08:24:29.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026391550505050505, "avg_duration_per_task_ms": 26392.646464646463, "avg_output_tokens_per_task": 2240.712121212121, "last_run_timestamp": "2026-10-01T08:37:17.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.918596129750983, "accuracy_std_dev": 0.013046134717055308, "run_count": 12, "total_questions": 2289, "avg_cost_per_task_usd": 0.12497408082131936, "avg_duration_per_task_ms": 123328.45827872433, "avg_output_tokens_per_task": 8354.565749235475, "last_run_timestamp": "2026-10-10T08:22:13.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.91835, "accuracy_std_dev": 0.0070592047474674425, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09014416835016834, "avg_duration_per_task_ms": 315664.6085858586, "avg_output_tokens_per_task": 25003.872053872055, "last_run_timestamp": "2026-10-10T08:22:27.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9172897562692477, "accuracy_std_dev": 0.03762026976984024, "run_count": 12, "total_questions": 2273, "avg_cost_per_task_usd": 0.12029984205895294, "avg_duration_per_task_ms": 200092.88737351517, "avg_output_tokens_per_task": 9308.563572371315, "last_run_timestamp": "2026-10-10T08:25:06.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9158840305164319, "accuracy_std_dev": 0.007323011048053776, "run_count": 11, "total_questions": 2130, "avg_cost_per_task_usd": 0.14128096619718308, "avg_duration_per_task_ms": 291601.6774647887, "avg_output_tokens_per_task": 9258.633333333333, "last_run_timestamp": "2026-10-10T08:23:05.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9155198394495413, "accuracy_std_dev": 0.012423863218230725, "run_count": 7, "total_questions": 872, "avg_cost_per_task_usd": 0.10930780733944953, "avg_duration_per_task_ms": 186343.69036697247, "avg_output_tokens_per_task": 7334.917431192661, "last_run_timestamp": "2026-10-10T08:22:26.808Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9152635, "accuracy_std_dev": 0.003022191630257765, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02597409595959596, "avg_duration_per_task_ms": 19482.909090909092, "avg_output_tokens_per_task": 2237.35101010101, "last_run_timestamp": "2026-10-10T08:24:21.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9145621666666666, "accuracy_std_dev": 0.012656594788444839, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.033476913720538716, "avg_duration_per_task_ms": 229462.40782828283, "avg_output_tokens_per_task": 27719.715488215486, "last_run_timestamp": "2026-10-10T16:22:04.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9141412857142857, "accuracy_std_dev": 0.006612511252256941, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11183320346320347, "avg_duration_per_task_ms": 640230.5497835497, "avg_output_tokens_per_task": 40187.943001443, "last_run_timestamp": "2026-10-10T08:14:42.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9131312, "accuracy_std_dev": 0.00943941759644102, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.15941067777777776, "avg_duration_per_task_ms": 524125.72525252524, "avg_output_tokens_per_task": 39938.53333333333, "last_run_timestamp": "2026-10-10T08:21:33.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.9128786136363636, "accuracy_std_dev": 0.01857399999999998, "run_count": 2, "total_questions": 352, "avg_cost_per_task_usd": 0.13041307102272726, "avg_duration_per_task_ms": 1907977.3181818181, "avg_output_tokens_per_task": 51917.37784090909, "last_run_timestamp": "2026-10-10T08:22:15.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.912456052026013, "accuracy_std_dev": 0.03101228282566373, "run_count": 21, "total_questions": 3998, "avg_cost_per_task_usd": 0.14550272386193094, "avg_duration_per_task_ms": 428036.3906953477, "avg_output_tokens_per_task": 8925.227113556779, "last_run_timestamp": "2026-10-10T08:15:00.283Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9117844, "accuracy_std_dev": 0.006079420781620585, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03790721616161616, "avg_duration_per_task_ms": 129456.97070707071, "avg_output_tokens_per_task": 28484.87575757576, "last_run_timestamp": "2026-10-03T09:28:48.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9106839879573877, "accuracy_std_dev": 0.01509499450450079, "run_count": 11, "total_questions": 2159, "avg_cost_per_task_usd": 0.18564463547938861, "avg_duration_per_task_ms": 504241.5956461325, "avg_output_tokens_per_task": 12306.873089393237, "last_run_timestamp": "2026-10-10T08:15:01.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9094274, "accuracy_std_dev": 0.002474202384608023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07117444141414142, "avg_duration_per_task_ms": 135218.17474747475, "avg_output_tokens_per_task": 27695.128282828282, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9093359666329627, "accuracy_std_dev": 0.004225510176298254, "run_count": 10, "total_questions": 1978, "avg_cost_per_task_usd": 0.03006448432760364, "avg_duration_per_task_ms": 457008.77856420627, "avg_output_tokens_per_task": 31093.431243680487, "last_run_timestamp": "2026-10-10T16:21:14.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0054125, "avg_duration_per_task_ms": 235839.91414141413, "avg_output_tokens_per_task": 10749.717171717171, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.9080194545454546, "accuracy_std_dev": 0.016023878405586615, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.13567432598714418, "avg_duration_per_task_ms": 551540.5821854912, "avg_output_tokens_per_task": 39959.97474747475, "last_run_timestamp": "2026-10-10T08:34:36.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9075092967032967, "accuracy_std_dev": 0.0008110000000000062, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.15048144505494504, "avg_duration_per_task_ms": 345399.21428571426, "avg_output_tokens_per_task": 38131.52197802198, "last_run_timestamp": "2026-10-03T10:04:03.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9073368212927757, "accuracy_std_dev": 0.025244605272986335, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.03730512885509083, "avg_duration_per_task_ms": 249194.64807773553, "avg_output_tokens_per_task": 30447.69877482045, "last_run_timestamp": "2026-10-10T16:20:06.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9071381724573594, "accuracy_std_dev": 0.03371213456235009, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.015826173089071385, "avg_duration_per_task_ms": 64047.12571067593, "avg_output_tokens_per_task": 3604.423246999368, "last_run_timestamp": "2026-09-30T03:11:21.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9068924337714864, "accuracy_std_dev": 0.016448572625003058, "run_count": 10, "total_questions": 1978, "avg_cost_per_task_usd": 0.035087561172901925, "avg_duration_per_task_ms": 328169.32204246713, "avg_output_tokens_per_task": 29201.10465116279, "last_run_timestamp": "2026-10-10T16:22:15.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9061828181818181, "accuracy_std_dev": 0.02164377168365568, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03816088016528926, "avg_duration_per_task_ms": 270123.5055096419, "avg_output_tokens_per_task": 31994.34618916437, "last_run_timestamp": "2026-10-10T16:23:09.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9056252185030218, "accuracy_std_dev": 0.011533357236316561, "run_count": 11, "total_questions": 2151, "avg_cost_per_task_usd": 0.028380999070199905, "avg_duration_per_task_ms": 289429.54486285447, "avg_output_tokens_per_task": 23515.902370990236, "last_run_timestamp": "2026-10-10T16:20:06.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.9055708181818182, "accuracy_std_dev": 0.027871027429338564, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.042199853076216716, "avg_duration_per_task_ms": 291574.9090909091, "avg_output_tokens_per_task": 32546.853076216714, "last_run_timestamp": "2026-10-10T16:22:42.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9055076503284487, "accuracy_std_dev": 0.05364913641103649, "run_count": 10, "total_questions": 1979, "avg_cost_per_task_usd": 0.13999162253663464, "avg_duration_per_task_ms": 293935.19656392117, "avg_output_tokens_per_task": 10089.026275896918, "last_run_timestamp": "2026-10-10T08:21:49.772Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9050427041719343, "accuracy_std_dev": 0.01882757447203436, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.03372518246944795, "avg_duration_per_task_ms": 328089.9827222925, "avg_output_tokens_per_task": 29343.05309734513, "last_run_timestamp": "2026-10-10T16:21:42.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9046013333333333, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03432480808080808, "avg_duration_per_task_ms": 135428.62121212122, "avg_output_tokens_per_task": 28424.8265993266, "last_run_timestamp": "2026-10-10T08:22:10.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.904377, "accuracy_std_dev": 0.009122177678602841, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.033627452525252524, "avg_duration_per_task_ms": 247053.7898989899, "avg_output_tokens_per_task": 27843.98181818182, "last_run_timestamp": "2026-10-10T08:11:33.638Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9038719, "accuracy_std_dev": 0.022667207355340432, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.15332785707070706, "avg_duration_per_task_ms": 585989.541919192, "avg_output_tokens_per_task": 38728.24494949495, "last_run_timestamp": "2026-10-10T08:21:19.694Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9036284693670886, "accuracy_std_dev": 0.014048473697879075, "run_count": 10, "total_questions": 1975, "avg_cost_per_task_usd": 0.02646089265822785, "avg_duration_per_task_ms": 194010.8364556962, "avg_output_tokens_per_task": 25873.973164556963, "last_run_timestamp": "2026-10-10T08:25:15.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9034791666666666, "accuracy_std_dev": 0.007995417675491095, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022680801346801345, "avg_duration_per_task_ms": 208477.4335016835, "avg_output_tokens_per_task": 29237.281986531987, "last_run_timestamp": "2026-10-10T08:29:50.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9032354285714285, "accuracy_std_dev": 0.022634937934174912, "run_count": 9, "total_questions": 1638, "avg_cost_per_task_usd": 0.1526553308913309, "avg_duration_per_task_ms": 567262.6489621489, "avg_output_tokens_per_task": 37775.336996337, "last_run_timestamp": "2026-10-10T08:21:36.296Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9031220000000001, "accuracy_std_dev": 0.04051469049167676, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.14198908172635447, "avg_duration_per_task_ms": 209617.58539944905, "avg_output_tokens_per_task": 9568.357208448118, "last_run_timestamp": "2026-10-10T08:25:16.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902918, "accuracy_std_dev": 0.0007933738084913016, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14672305387205387, "avg_duration_per_task_ms": 133620.78956228957, "avg_output_tokens_per_task": 9652.131313131313, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.9027134284064666, "accuracy_std_dev": 0.015276438772029145, "run_count": 9, "total_questions": 1732, "avg_cost_per_task_usd": 0.014871293879907621, "avg_duration_per_task_ms": 274837.21131639724, "avg_output_tokens_per_task": 18283.435912240184, "last_run_timestamp": "2026-10-10T08:14:58.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9024151322662172, "accuracy_std_dev": 0.007540900139682681, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.0312832594776748, "avg_duration_per_task_ms": 159175.5240101095, "avg_output_tokens_per_task": 29122.362257792756, "last_run_timestamp": "2026-10-10T08:15:01.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9023917412834764, "accuracy_std_dev": 0.031521333558242756, "run_count": 10, "total_questions": 1979, "avg_cost_per_task_usd": 0.10391216371905002, "avg_duration_per_task_ms": 566571.5058110156, "avg_output_tokens_per_task": 39085.76503284487, "last_run_timestamp": "2026-10-10T08:36:06.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.902357, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0541827138047138, "avg_duration_per_task_ms": 33856.301346801345, "avg_output_tokens_per_task": 2498.9175084175085, "last_run_timestamp": "2026-10-10T08:22:13.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9023568333333333, "accuracy_std_dev": 0.00927208580303744, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03514278535353535, "avg_duration_per_task_ms": 197697.9335016835, "avg_output_tokens_per_task": 29114.67845117845, "last_run_timestamp": "2026-10-10T08:27:28.674Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9017953333333333, "accuracy_std_dev": 0.0041993868070892585, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03234963973063973, "avg_duration_per_task_ms": 866618.7946127946, "avg_output_tokens_per_task": 36837.547138047135, "last_run_timestamp": "2026-10-10T08:37:36.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.900939869848156, "accuracy_std_dev": 0.007380306298604699, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.01601901446131598, "avg_duration_per_task_ms": 443238.66449746926, "avg_output_tokens_per_task": 28948.347071583514, "last_run_timestamp": "2026-10-10T08:23:04.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.90067325, "accuracy_std_dev": 0.005350183771376443, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.20474679772727275, "avg_duration_per_task_ms": 98144.71237373738, "avg_output_tokens_per_task": 7846.206818181819, "last_run_timestamp": "2026-10-10T08:22:41.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.90067325, "accuracy_std_dev": 0.0031494064817835163, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.053791821969696965, "avg_duration_per_task_ms": 27301.564393939392, "avg_output_tokens_per_task": 2474.441919191919, "last_run_timestamp": "2026-10-10T08:23:49.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.900619, "accuracy_std_dev": 0.005948071674514441, "run_count": 31, "total_questions": 6138, "avg_cost_per_task_usd": 0.21661038025415447, "avg_duration_per_task_ms": 94471.30303030302, "avg_output_tokens_per_task": 7896.7298794395565, "last_run_timestamp": "2026-10-10T08:22:25.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.9005056897133222, "accuracy_std_dev": 0.017065352280115537, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.1164296958965711, "avg_duration_per_task_ms": 597873.8173130973, "avg_output_tokens_per_task": 36394.80157391793, "last_run_timestamp": "2026-10-10T08:14:22.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9003566096256683, "accuracy_std_dev": 0.009294020279657108, "run_count": 11, "total_questions": 1870, "avg_cost_per_task_usd": 0.023890174331550804, "avg_duration_per_task_ms": 191466.34438502675, "avg_output_tokens_per_task": 19764.336898395723, "last_run_timestamp": "2026-10-10T16:21:33.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8999248888888889, "accuracy_std_dev": 0.006354339085213833, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15479229124579125, "avg_duration_per_task_ms": 130090.1857463524, "avg_output_tokens_per_task": 9547.675645342311, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.8998315, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11587846212121213, "avg_duration_per_task_ms": 577782.9368686868, "avg_output_tokens_per_task": 46174.61616161616, "last_run_timestamp": "2026-10-10T08:11:30.726Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14109027272727273, "avg_duration_per_task_ms": 131303, "avg_output_tokens_per_task": 9278.422558922559, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0412280202020202, "avg_duration_per_task_ms": 141835.40572390571, "avg_output_tokens_per_task": 27211.210437710437, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8995507777777776, "accuracy_std_dev": 0.004558884848483037, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.2283120224466891, "avg_duration_per_task_ms": 110024.31481481482, "avg_output_tokens_per_task": 8923.117845117846, "last_run_timestamp": "2026-10-10T08:24:16.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8994632500000004, "accuracy_std_dev": 0.0063860840847501914, "run_count": 32, "total_questions": 6336, "avg_cost_per_task_usd": 0.22312757212752524, "avg_duration_per_task_ms": 94690.671875, "avg_output_tokens_per_task": 8125.733585858586, "last_run_timestamp": "2026-10-10T08:26:06.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8994013674033149, "accuracy_std_dev": 0.0057226206055478665, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.01418124585635359, "avg_duration_per_task_ms": 183996.83977900553, "avg_output_tokens_per_task": 26261.59944751381, "last_run_timestamp": "2026-10-10T08:22:32.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8992703333333333, "accuracy_std_dev": 0.004703538142245193, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05869316456228956, "avg_duration_per_task_ms": 43102.90572390572, "avg_output_tokens_per_task": 2536.3127104377104, "last_run_timestamp": "2026-10-10T08:37:23.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.898999033256351, "accuracy_std_dev": 0.018913767762537594, "run_count": 11, "total_questions": 2165, "avg_cost_per_task_usd": 0.1172629778290993, "avg_duration_per_task_ms": 375593.448960739, "avg_output_tokens_per_task": 8022.391224018475, "last_run_timestamp": "2026-10-10T08:28:02.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.005188652155184398, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03249949747474747, "avg_duration_per_task_ms": 137621.66666666666, "avg_output_tokens_per_task": 26156.715909090908, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.898989625, "accuracy_std_dev": 0.00564651885982635, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.057617888257575745, "avg_duration_per_task_ms": 25880.185606060608, "avg_output_tokens_per_task": 2525.5625, "last_run_timestamp": "2026-10-10T08:22:11.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna", "model_slug": "openai/gpt-6-luna", "model_name": "GPT-6-Luna", "provider_name": "auto-routing", "accuracy": 0.898477, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.003442197969543147, "avg_duration_per_task_ms": 73312.30456852792, "avg_output_tokens_per_task": 6831.9390862944165, "last_run_timestamp": "2026-10-08T13:29:57.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8984286666666665, "accuracy_std_dev": 0.002099515235053612, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03384256734006734, "avg_duration_per_task_ms": 896786.5488215488, "avg_output_tokens_per_task": 38532.71885521885, "last_run_timestamp": "2026-10-10T08:14:49.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8982883333333332, "accuracy_std_dev": 0.003854920542487059, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0574033244949495, "avg_duration_per_task_ms": 25094.159511784514, "avg_output_tokens_per_task": 2482.843434343434, "last_run_timestamp": "2026-10-10T08:25:06.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8982242269380685, "accuracy_std_dev": 0.025718218408254386, "run_count": 13, "total_questions": 2309, "avg_cost_per_task_usd": 0.2508324629709831, "avg_duration_per_task_ms": 414948.73408401903, "avg_output_tokens_per_task": 46979.851017756606, "last_run_timestamp": "2026-10-10T08:40:32.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8978673333333335, "accuracy_std_dev": 0.021639482631420633, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.08683191161616162, "avg_duration_per_task_ms": 208079.4877946128, "avg_output_tokens_per_task": 29133.21127946128, "last_run_timestamp": "2026-10-10T16:33:12.083Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.8978673333333335, "accuracy_std_dev": 0.0103171593420325, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.12474947474747476, "avg_duration_per_task_ms": 901534.3451178451, "avg_output_tokens_per_task": 49639.851851851854, "last_run_timestamp": "2026-10-10T08:29:32.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.897306, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.12030833838383839, "avg_duration_per_task_ms": 846579.9494949495, "avg_output_tokens_per_task": 47921.80303030303, "last_run_timestamp": "2026-10-10T08:14:41.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8971503581183612, "accuracy_std_dev": 0.02073357692150583, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.12949166211431462, "avg_duration_per_task_ms": 522272.583712696, "avg_output_tokens_per_task": 38320.582701062216, "last_run_timestamp": "2026-10-10T08:21:10.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8969694999999998, "accuracy_std_dev": 0.01185165437607762, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.2005278484848485, "avg_duration_per_task_ms": 96563.55505050505, "avg_output_tokens_per_task": 7804.548484848485, "last_run_timestamp": "2026-10-10T08:14:40.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8969322222222222, "accuracy_std_dev": 0.0084278559785429, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1972121043771044, "avg_duration_per_task_ms": 95091.35016835017, "avg_output_tokens_per_task": 7682.2547699214365, "last_run_timestamp": "2026-10-10T08:22:27.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8968568964143425, "accuracy_std_dev": 0.005278733293840445, "run_count": 4, "total_questions": 753, "avg_cost_per_task_usd": 0.15754132403718457, "avg_duration_per_task_ms": 84099.57768924303, "avg_output_tokens_per_task": 9420.674634794157, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8967648379746836, "accuracy_std_dev": 0.007137443646401371, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.018252328270042195, "avg_duration_per_task_ms": 503009.7375527426, "avg_output_tokens_per_task": 19743.828691983123, "last_run_timestamp": "2026-10-10T08:22:09.106Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8967118381024095, "accuracy_std_dev": 0.027051445539009556, "run_count": 8, "total_questions": 1328, "avg_cost_per_task_usd": 0.026119229668674696, "avg_duration_per_task_ms": 760619.1935240964, "avg_output_tokens_per_task": 23698.94503012048, "last_run_timestamp": "2026-10-10T16:24:01.594Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8963888977035491, "accuracy_std_dev": 0.007117042546266783, "run_count": 22, "total_questions": 4311, "avg_cost_per_task_usd": 0.12574247158431917, "avg_duration_per_task_ms": 57691.72442588727, "avg_output_tokens_per_task": 4730.365576432382, "last_run_timestamp": "2026-10-10T08:28:02.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.8962022556962025, "accuracy_std_dev": 0.0022689999999999655, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.020123096202531644, "avg_duration_per_task_ms": 653583.2962025317, "avg_output_tokens_per_task": 42659.516455696205, "last_run_timestamp": "2026-10-10T00:00:04.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8957578329718004, "accuracy_std_dev": 0.016809562938053055, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.023020330802603038, "avg_duration_per_task_ms": 236991.16630513375, "avg_output_tokens_per_task": 26771.403470715835, "last_run_timestamp": "2026-10-10T08:26:17.965Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8956230000000002, "accuracy_std_dev": 0.007273551310513056, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00771163468013468, "avg_duration_per_task_ms": 254636.3367003367, "avg_output_tokens_per_task": 27268.55387205387, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8950130708661417, "accuracy_std_dev": 0.00891334037763057, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.048151682414698166, "avg_duration_per_task_ms": 43476.24934383202, "avg_output_tokens_per_task": 3503.036745406824, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8950104545454546, "accuracy_std_dev": 0.007224079964870452, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12169120569329661, "avg_duration_per_task_ms": 58147.77502295684, "avg_output_tokens_per_task": 4649.89898989899, "last_run_timestamp": "2026-10-10T08:30:55.092Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8949752307692306, "accuracy_std_dev": 0.005999147723492375, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.20542784071484071, "avg_duration_per_task_ms": 100020.3480963481, "avg_output_tokens_per_task": 8004.257187257187, "last_run_timestamp": "2026-10-10T08:24:37.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.894781, "accuracy_std_dev": 0.004835703206359931, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.054542488636363635, "avg_duration_per_task_ms": 32603.121212121212, "avg_output_tokens_per_task": 2523.126262626263, "last_run_timestamp": "2026-10-10T08:22:14.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.894653484848485, "accuracy_std_dev": 0.00729743750841102, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.20388223569023572, "avg_duration_per_task_ms": 70408.00857055403, "avg_output_tokens_per_task": 7423.640189776554, "last_run_timestamp": "2026-10-10T08:29:34.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.894286846153846, "accuracy_std_dev": 0.010109118939518566, "run_count": 15, "total_questions": 2964, "avg_cost_per_task_usd": 0.013039793522267208, "avg_duration_per_task_ms": 301749.21187584347, "avg_output_tokens_per_task": 27435.895074224023, "last_run_timestamp": "2026-10-10T08:29:49.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8941264444444444, "accuracy_std_dev": 0.009636502052221474, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05178651795735129, "avg_duration_per_task_ms": 45445.343434343435, "avg_output_tokens_per_task": 3671.5897867564536, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8939462206896551, "accuracy_std_dev": 0.00852872826887441, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.22604011034482757, "avg_duration_per_task_ms": 108518.86942528735, "avg_output_tokens_per_task": 8818.531954022988, "last_run_timestamp": "2026-10-10T08:14:40.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8936586666666667, "accuracy_std_dev": 0.02110358604334552, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.026894239057239058, "avg_duration_per_task_ms": 149425.4292929293, "avg_output_tokens_per_task": 27645.62373737374, "last_run_timestamp": "2026-10-10T08:22:18.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8933884681122448, "accuracy_std_dev": 0.01364679809916505, "run_count": 16, "total_questions": 3136, "avg_cost_per_task_usd": 0.0038534390943877547, "avg_duration_per_task_ms": 324360.03539540817, "avg_output_tokens_per_task": 21118.27838010204, "last_run_timestamp": "2026-10-10T08:21:47.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.8933779999999999, "accuracy_std_dev": 0.008285503967774051, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11363387037037037, "avg_duration_per_task_ms": 879366.8148148148, "avg_output_tokens_per_task": 45257.41750841751, "last_run_timestamp": "2026-10-10T08:22:57.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.89267675, "accuracy_std_dev": 0.006979218773437292, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037834488636363635, "avg_duration_per_task_ms": 50710.07575757576, "avg_output_tokens_per_task": 3480.1502525252527, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8913205555555556, "accuracy_std_dev": 0.01785381330641292, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14202826936026933, "avg_duration_per_task_ms": 616787.0375982042, "avg_output_tokens_per_task": 35812.23737373737, "last_run_timestamp": "2026-10-10T08:21:47.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8912990824295012, "accuracy_std_dev": 0.011335972984040578, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.20838120318148953, "avg_duration_per_task_ms": 100499.53289949386, "avg_output_tokens_per_task": 8107.096167751265, "last_run_timestamp": "2026-10-10T08:25:06.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8910347038626609, "accuracy_std_dev": 0.006181130251620641, "run_count": 4, "total_questions": 699, "avg_cost_per_task_usd": 0.01261281974248927, "avg_duration_per_task_ms": 330150.28755364806, "avg_output_tokens_per_task": 23768.37339055794, "last_run_timestamp": "2026-10-10T08:33:55.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8910211333333333, "accuracy_std_dev": 0.013724636800861273, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.01418473804713805, "avg_duration_per_task_ms": 211329.1047138047, "avg_output_tokens_per_task": 24782.838047138048, "last_run_timestamp": "2026-10-10T08:22:09.792Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.891002888888889, "accuracy_std_dev": 0.0356098077018116, "run_count": 10, "total_questions": 1971, "avg_cost_per_task_usd": 0.14153233739218668, "avg_duration_per_task_ms": 796678.4495180112, "avg_output_tokens_per_task": 35436.016235413495, "last_run_timestamp": "2026-10-10T08:24:22.693Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8907346275957544, "accuracy_std_dev": 0.009380068570120645, "run_count": 33, "total_questions": 6501, "avg_cost_per_task_usd": 0.13349859406245196, "avg_duration_per_task_ms": 59227.62467312721, "avg_output_tokens_per_task": 4775.171665897554, "last_run_timestamp": "2026-10-10T08:22:14.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8905723333333333, "accuracy_std_dev": 0.021339306148253502, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13216584343434343, "avg_duration_per_task_ms": 251067.54713804714, "avg_output_tokens_per_task": 43934.003367003366, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.8902381885714286, "accuracy_std_dev": 0.0030799415984722714, "run_count": 4, "total_questions": 700, "avg_cost_per_task_usd": 0.03244278, "avg_duration_per_task_ms": 834708.2842857143, "avg_output_tokens_per_task": 37089.54, "last_run_timestamp": "2026-10-10T08:11:33.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8901512500000001, "accuracy_std_dev": 0.006979218773437337, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008867842171717173, "avg_duration_per_task_ms": 175715.7159090909, "avg_output_tokens_per_task": 31193.151515151516, "last_run_timestamp": "2026-10-10T08:26:16.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.890118616572311, "accuracy_std_dev": 0.05543710090986, "run_count": 10, "total_questions": 1943, "avg_cost_per_task_usd": 0.011268564590838909, "avg_duration_per_task_ms": 493758.26093669585, "avg_output_tokens_per_task": 21520.772516726713, "last_run_timestamp": "2026-10-10T08:48:02.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8900365454545454, "accuracy_std_dev": 0.008514741728465947, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.24217010583103765, "avg_duration_per_task_ms": 85203.68801652892, "avg_output_tokens_per_task": 9007.777777777777, "last_run_timestamp": "2026-10-10T08:24:21.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.005722799567422281, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.19800758417508418, "avg_duration_per_task_ms": 96141.14141414141, "avg_output_tokens_per_task": 7695.558922558923, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8900110000000001, "accuracy_std_dev": 0.0055552329084086844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2164912121212121, "avg_duration_per_task_ms": 104660.01851851853, "avg_output_tokens_per_task": 8434.90404040404, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8898569891067537, "accuracy_std_dev": 0.020559295245207023, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.19048945533769066, "avg_duration_per_task_ms": 1402381.5344952797, "avg_output_tokens_per_task": 55920.59404502542, "last_run_timestamp": "2026-10-10T16:21:04.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8893492596685083, "accuracy_std_dev": 0.01020692558839056, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.20542008609576426, "avg_duration_per_task_ms": 100076.91850828729, "avg_output_tokens_per_task": 7990.411602209945, "last_run_timestamp": "2026-10-10T08:21:58.851Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8888376274149035, "accuracy_std_dev": 0.008182609561727207, "run_count": 11, "total_questions": 2174, "avg_cost_per_task_usd": 0.015841472401103954, "avg_duration_per_task_ms": 163351.5689972401, "avg_output_tokens_per_task": 26304.647194112236, "last_run_timestamp": "2026-10-10T08:22:23.779Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8884680000000001, "accuracy_std_dev": 0.008615114711946651, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02470814393939394, "avg_duration_per_task_ms": 276879.46338383836, "avg_output_tokens_per_task": 29963.959595959597, "last_run_timestamp": "2026-10-10T08:22:50.656Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8878433783257624, "accuracy_std_dev": 0.020080093052184153, "run_count": 8, "total_questions": 1541, "avg_cost_per_task_usd": 0.155311282284231, "avg_duration_per_task_ms": 605967.6262167423, "avg_output_tokens_per_task": 42247.91953277093, "last_run_timestamp": "2026-10-10T08:22:51.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8877663333333334, "accuracy_std_dev": 0.005555502277522277, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03440870875420875, "avg_duration_per_task_ms": 50195.98148148148, "avg_output_tokens_per_task": 3309.2491582491584, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8873838777089784, "accuracy_std_dev": 0.02655499827504712, "run_count": 7, "total_questions": 1292, "avg_cost_per_task_usd": 0.009403260061919503, "avg_duration_per_task_ms": 395325.9891640867, "avg_output_tokens_per_task": 19091.013157894737, "last_run_timestamp": "2026-10-10T08:23:40.493Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8872052500000001, "accuracy_std_dev": 0.0031496737430248356, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.012731213383838385, "avg_duration_per_task_ms": 417944.53661616164, "avg_output_tokens_per_task": 23604.305555555555, "last_run_timestamp": "2026-10-10T08:23:04.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.887205, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007206747474747474, "avg_duration_per_task_ms": 979612.8939393939, "avg_output_tokens_per_task": 42001.40909090909, "last_run_timestamp": "2026-10-10T16:19:50.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8868686, "accuracy_std_dev": 0.01990232228258803, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.029374592929292932, "avg_duration_per_task_ms": 212707.40101010102, "avg_output_tokens_per_task": 27438.87777777778, "last_run_timestamp": "2026-10-10T08:11:30.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8865611646002564, "accuracy_std_dev": 0.008235396939431653, "run_count": 12, "total_questions": 2339, "avg_cost_per_task_usd": 0.09613296280461736, "avg_duration_per_task_ms": 530353.1603249252, "avg_output_tokens_per_task": 29119.90594271056, "last_run_timestamp": "2026-10-10T08:48:04.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.8860826666666668, "accuracy_std_dev": 0.009654676391377534, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.006178414141414142, "avg_duration_per_task_ms": 550604.5437710438, "avg_output_tokens_per_task": 33932.367003367006, "last_run_timestamp": "2026-10-10T08:22:13.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8855269617834396, "accuracy_std_dev": 0.01525692535211471, "run_count": 5, "total_questions": 942, "avg_cost_per_task_usd": 0.03748663375796179, "avg_duration_per_task_ms": 257130.8949044586, "avg_output_tokens_per_task": 34302.59554140127, "last_run_timestamp": "2026-10-10T08:22:50.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8855220000000001, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1415003787878788, "avg_duration_per_task_ms": 184472.2676767677, "avg_output_tokens_per_task": 47043.42424242424, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8853288347386172, "accuracy_std_dev": 0.0037367403204159573, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.032569026981450255, "avg_duration_per_task_ms": 806245.0269814503, "avg_output_tokens_per_task": 37423.73524451939, "last_run_timestamp": "2026-10-10T08:21:50.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8853190481927712, "accuracy_std_dev": 0.005795847495189989, "run_count": 4, "total_questions": 747, "avg_cost_per_task_usd": 0.0097919718875502, "avg_duration_per_task_ms": 50542.31994645247, "avg_output_tokens_per_task": 7272.5394912985275, "last_run_timestamp": "2026-10-01T19:09:12.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8850689656218401, "accuracy_std_dev": 0.014411932672615443, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.026149528816986855, "avg_duration_per_task_ms": 355797.47320525785, "avg_output_tokens_per_task": 26995.41152679474, "last_run_timestamp": "2026-10-10T08:22:45.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8850408571428571, "accuracy_std_dev": 0.015130548440900905, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00522727344877345, "avg_duration_per_task_ms": 245222.45887445888, "avg_output_tokens_per_task": 28404.085137085138, "last_run_timestamp": "2026-10-10T16:19:44.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.8848482000000001, "accuracy_std_dev": 0.003626295707743655, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.029178751515151518, "avg_duration_per_task_ms": 89030.6888888889, "avg_output_tokens_per_task": 24171.247474747473, "last_run_timestamp": "2026-10-10T08:26:07.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8848054784010315, "accuracy_std_dev": 0.03136720764189888, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.009929504835589941, "avg_duration_per_task_ms": 329283.4983881367, "avg_output_tokens_per_task": 25727.783365570598, "last_run_timestamp": "2026-10-10T08:21:36.060Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8846618235507244, "accuracy_std_dev": 0.01231654174229908, "run_count": 14, "total_questions": 2760, "avg_cost_per_task_usd": 0.007433793115942029, "avg_duration_per_task_ms": 452053.6221014493, "avg_output_tokens_per_task": 26553.594202898552, "last_run_timestamp": "2026-10-10T08:23:40.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8845116000000001, "accuracy_std_dev": 0.005694030895595855, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.028268917171717173, "avg_duration_per_task_ms": 139965.10808080807, "avg_output_tokens_per_task": 23399.145454545454, "last_run_timestamp": "2026-10-10T08:22:26.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8842973636363637, "accuracy_std_dev": 0.009633322699432706, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.02382341873278237, "avg_duration_per_task_ms": 351638.5702479339, "avg_output_tokens_per_task": 26165.873278236915, "last_run_timestamp": "2026-10-10T08:28:12.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.884269595505618, "accuracy_std_dev": 0.010583520080008851, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.018860816853932583, "avg_duration_per_task_ms": 74171.62584269662, "avg_output_tokens_per_task": 7862.741573033708, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.883922059602649, "accuracy_std_dev": 0.013047095505854499, "run_count": 6, "total_questions": 906, "avg_cost_per_task_usd": 0.007205040838852097, "avg_duration_per_task_ms": 953103.2119205298, "avg_output_tokens_per_task": 28596.360927152316, "last_run_timestamp": "2026-10-10T08:21:47.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8838929412284579, "accuracy_std_dev": 0.013075471353610644, "run_count": 23, "total_questions": 4526, "avg_cost_per_task_usd": 0.19182242841361027, "avg_duration_per_task_ms": 375443.09456473705, "avg_output_tokens_per_task": 31243.90676093681, "last_run_timestamp": "2026-10-10T08:22:13.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8838383333333335, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13180945791245793, "avg_duration_per_task_ms": 248970.3754208754, "avg_output_tokens_per_task": 43817.097643097644, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8838382499999999, "accuracy_std_dev": 0.01653796711169484, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.08040183459595959, "avg_duration_per_task_ms": 318120.2462121212, "avg_output_tokens_per_task": 28808.45580808081, "last_run_timestamp": "2026-10-10T08:22:11.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8838381818181819, "accuracy_std_dev": 0.019315258494108086, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.025690312672176308, "avg_duration_per_task_ms": 215061.82185491276, "avg_output_tokens_per_task": 25361.899449035813, "last_run_timestamp": "2026-10-10T08:21:47.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8838381111111112, "accuracy_std_dev": 0.013444611638078868, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.18439962401795731, "avg_duration_per_task_ms": 304152.4012345679, "avg_output_tokens_per_task": 46284.17845117845, "last_run_timestamp": "2026-10-10T08:14:46.816Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8836850909090909, "accuracy_std_dev": 0.01588134704301956, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.11592730165289256, "avg_duration_per_task_ms": 601761.9191919192, "avg_output_tokens_per_task": 30287.78328741965, "last_run_timestamp": "2026-10-10T08:14:58.268Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.883660005882353, "accuracy_std_dev": 0.0111840129195372, "run_count": 8, "total_questions": 1530, "avg_cost_per_task_usd": 0.06956762679738562, "avg_duration_per_task_ms": 311666.74901960784, "avg_output_tokens_per_task": 19224.162745098038, "last_run_timestamp": "2026-10-10T16:19:40.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8835341342512908, "accuracy_std_dev": 0.0025738249530394865, "run_count": 3, "total_questions": 581, "avg_cost_per_task_usd": 0.12498561962134253, "avg_duration_per_task_ms": 454716.72461273667, "avg_output_tokens_per_task": 36016.14457831325, "last_run_timestamp": "2026-10-10T16:20:08.973Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8834197726683938, "accuracy_std_dev": 0.04493897289102186, "run_count": 8, "total_questions": 1544, "avg_cost_per_task_usd": 0.05442232836787564, "avg_duration_per_task_ms": 582124.3406735752, "avg_output_tokens_per_task": 18408.36658031088, "last_run_timestamp": "2026-10-10T08:21:48.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8833552031872511, "accuracy_std_dev": 0.02275586572183091, "run_count": 4, "total_questions": 753, "avg_cost_per_task_usd": 0.015250491367861886, "avg_duration_per_task_ms": 298505.31341301464, "avg_output_tokens_per_task": 26478.50996015936, "last_run_timestamp": "2026-10-10T08:14:42.775Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8828329620853078, "accuracy_std_dev": 0.07234827378689748, "run_count": 14, "total_questions": 2532, "avg_cost_per_task_usd": 0.009765114533965244, "avg_duration_per_task_ms": 152136.68799368088, "avg_output_tokens_per_task": 23748.714060031594, "last_run_timestamp": "2026-10-10T08:25:04.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8824462686258934, "accuracy_std_dev": 0.010300883361604722, "run_count": 33, "total_questions": 6295, "avg_cost_per_task_usd": 0.13718419205718824, "avg_duration_per_task_ms": 62608.661318506754, "avg_output_tokens_per_task": 4975.615250198571, "last_run_timestamp": "2026-10-10T08:22:10.164Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8823811431670283, "accuracy_std_dev": 0.01015848669631376, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.00803116413593637, "avg_duration_per_task_ms": 247598.04772234274, "avg_output_tokens_per_task": 28350.216919739698, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8820143333333333, "accuracy_std_dev": 0.012187637016893623, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.00562553324915825, "avg_duration_per_task_ms": 391236.59764309763, "avg_output_tokens_per_task": 29487.287037037036, "last_run_timestamp": "2026-10-10T08:22:27.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.02691867172330274, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09661549711399711, "avg_duration_per_task_ms": 473415.0324675325, "avg_output_tokens_per_task": 35715.57359307359, "last_run_timestamp": "2026-10-10T16:26:22.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8811446, "accuracy_std_dev": 0.0062631755715451785, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.028506177777777777, "avg_duration_per_task_ms": 118218.53838383839, "avg_output_tokens_per_task": 23561.454545454544, "last_run_timestamp": "2026-10-10T08:30:31.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8810132745849297, "accuracy_std_dev": 0.029062612893329138, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.03828848659003832, "avg_duration_per_task_ms": 742925.4495530013, "avg_output_tokens_per_task": 25587.997445721583, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.8808888960000001, "accuracy_std_dev": 0.009359000000000006, "run_count": 2, "total_questions": 375, "avg_cost_per_task_usd": 0.10911519200000001, "avg_duration_per_task_ms": 1097247.664, "avg_output_tokens_per_task": 43639.90933333334, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8808079825757577, "accuracy_std_dev": 0.02428919623862362, "run_count": 7, "total_questions": 1320, "avg_cost_per_task_usd": 0.10256016742424243, "avg_duration_per_task_ms": 656305.7287878788, "avg_output_tokens_per_task": 29271.418181818182, "last_run_timestamp": "2026-10-10T08:22:26.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8805554416666667, "accuracy_std_dev": 0.010188565454298696, "run_count": 11, "total_questions": 2160, "avg_cost_per_task_usd": 0.005995147685185185, "avg_duration_per_task_ms": 391554.91064814816, "avg_output_tokens_per_task": 18833.464351851853, "last_run_timestamp": "2026-10-10T08:14:58.180Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.880471, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.042269898989898996, "avg_duration_per_task_ms": 255330.39562289562, "avg_output_tokens_per_task": 37185.93771043771, "last_run_timestamp": "2026-10-10T08:21:41.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8794386469072164, "accuracy_std_dev": 0.018595844993461137, "run_count": 12, "total_questions": 2328, "avg_cost_per_task_usd": 0.12945956872852238, "avg_duration_per_task_ms": 63011.24012027492, "avg_output_tokens_per_task": 4952.514604810996, "last_run_timestamp": "2026-10-10T08:14:49.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8793878407750632, "accuracy_std_dev": 0.007247192184257008, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.015630656276326874, "avg_duration_per_task_ms": 322819.4338668913, "avg_output_tokens_per_task": 28000.395956192082, "last_run_timestamp": "2026-10-10T08:36:28.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.879189, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.020723571428571428, "avg_duration_per_task_ms": 141095.54497354498, "avg_output_tokens_per_task": 17734.21164021164, "last_run_timestamp": "2026-09-15T00:33:26.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8790546552363299, "accuracy_std_dev": 0.030844451113161732, "run_count": 6, "total_questions": 1079, "avg_cost_per_task_usd": 0.1346450278035218, "avg_duration_per_task_ms": 284974.0537534754, "avg_output_tokens_per_task": 31126.302131603337, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "Deepseek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010643959595959596, "avg_duration_per_task_ms": 127627.76767676767, "avg_output_tokens_per_task": 14277.949494949495, "last_run_timestamp": "2026-10-08T13:29:57.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Mimo V2.6 Pro", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01249009595959596, "avg_duration_per_task_ms": 315799.5808080808, "avg_output_tokens_per_task": 14226.873737373737, "last_run_timestamp": "2026-10-08T01:55:44.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8787876666666666, "accuracy_std_dev": 0.005991714074916775, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007907055555555554, "avg_duration_per_task_ms": 79561.398989899, "avg_output_tokens_per_task": 7834.653198653199, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8779461000000001, "accuracy_std_dev": 0.03428938607922281, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10926873686868688, "avg_duration_per_task_ms": 517475.14797979797, "avg_output_tokens_per_task": 35212.96313131313, "last_run_timestamp": "2026-10-10T08:22:15.909Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8772414110344827, "accuracy_std_dev": 0.0072833211636968214, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.11771683494252876, "avg_duration_per_task_ms": 751137.611954023, "avg_output_tokens_per_task": 30859.994942528734, "last_run_timestamp": "2026-10-10T08:22:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8772163628528975, "accuracy_std_dev": 0.0321266435991701, "run_count": 17, "total_questions": 3365, "avg_cost_per_task_usd": 0.14701722585438334, "avg_duration_per_task_ms": 392848.22288261517, "avg_output_tokens_per_task": 11120.41337295691, "last_run_timestamp": "2026-10-10T08:22:13.763Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8771433591105211, "accuracy_std_dev": 0.1534252530590106, "run_count": 17, "total_questions": 3013, "avg_cost_per_task_usd": 0.14607643743776966, "avg_duration_per_task_ms": 264965.2558911384, "avg_output_tokens_per_task": 10925.349817457683, "last_run_timestamp": "2026-10-10T08:37:55.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8771043333333333, "accuracy_std_dev": 0.01914541124017855, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008207621212121211, "avg_duration_per_task_ms": 369369.9638047138, "avg_output_tokens_per_task": 28948.207912457914, "last_run_timestamp": "2026-10-10T16:21:38.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8761782480719793, "accuracy_std_dev": 0.00788648157846706, "run_count": 4, "total_questions": 778, "avg_cost_per_task_usd": 0.0930879910025707, "avg_duration_per_task_ms": 616690.5848329049, "avg_output_tokens_per_task": 27406.845758354757, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8760934928027095, "accuracy_std_dev": 0.008565775498394134, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.11545865114309908, "avg_duration_per_task_ms": 580224.5131244708, "avg_output_tokens_per_task": 32981.541913632514, "last_run_timestamp": "2026-10-10T16:22:35.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.876032909090909, "accuracy_std_dev": 0.03183332512108125, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1551542842056933, "avg_duration_per_task_ms": 356025.97061524336, "avg_output_tokens_per_task": 35808.172635445364, "last_run_timestamp": "2026-10-10T08:22:25.674Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8757575000000001, "accuracy_std_dev": 0.017781528276557126, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.13141335, "avg_duration_per_task_ms": 63646.402525252524, "avg_output_tokens_per_task": 5036.028787878788, "last_run_timestamp": "2026-10-10T08:11:16.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8756554578651687, "accuracy_std_dev": 0.014258761571593646, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.0903369808988764, "avg_duration_per_task_ms": 218770.61797752808, "avg_output_tokens_per_task": 22903.363483146066, "last_run_timestamp": "2026-10-10T08:21:17.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8755794209860935, "accuracy_std_dev": 0.004386999116708366, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.12302590328697852, "avg_duration_per_task_ms": 915411.7926675095, "avg_output_tokens_per_task": 43199.48293299621, "last_run_timestamp": "2026-10-10T16:19:34.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.875421, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1538974696969697, "avg_duration_per_task_ms": 363407.6313131313, "avg_output_tokens_per_task": 34764.520202020205, "last_run_timestamp": "2026-07-18T08:26:40.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8754207500000001, "accuracy_std_dev": 0.014884023314306964, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11076118223905723, "avg_duration_per_task_ms": 245485.56776094277, "avg_output_tokens_per_task": 31501.30387205387, "last_run_timestamp": "2026-10-10T08:22:18.403Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8754207142857143, "accuracy_std_dev": 0.015996248780924358, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.22405389971139966, "avg_duration_per_task_ms": 1033226.1435786436, "avg_output_tokens_per_task": 56216.72438672439, "last_run_timestamp": "2026-10-10T16:29:33.708Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8749732275334607, "accuracy_std_dev": 0.01165560971968007, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.042417240917782026, "avg_duration_per_task_ms": 238116.32058636073, "avg_output_tokens_per_task": 32745.878903760356, "last_run_timestamp": "2026-10-10T08:28:02.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8747894999999999, "accuracy_std_dev": 0.005115959905042276, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0784861994949495, "avg_duration_per_task_ms": 339416.61994949495, "avg_output_tokens_per_task": 22314.78724747475, "last_run_timestamp": "2026-10-10T16:21:08.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8744587142857143, "accuracy_std_dev": 0.028125314066636473, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1966599458874459, "avg_duration_per_task_ms": 1101198.378787879, "avg_output_tokens_per_task": 58277.85137085137, "last_run_timestamp": "2026-10-10T16:22:34.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8743913028237584, "accuracy_std_dev": 0.02063518118543075, "run_count": 6, "total_questions": 1027, "avg_cost_per_task_usd": 0.1379305988315482, "avg_duration_per_task_ms": 596483.0662122688, "avg_output_tokens_per_task": 50109.39727361246, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8741992451960031, "accuracy_std_dev": 0.025720260433664145, "run_count": 7, "total_questions": 1301, "avg_cost_per_task_usd": 0.009777912375096079, "avg_duration_per_task_ms": 392901.29208301305, "avg_output_tokens_per_task": 21503.2475019216, "last_run_timestamp": "2026-10-10T08:23:18.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8741581250000001, "accuracy_std_dev": 0.007610645331335265, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.049696884469696966, "avg_duration_per_task_ms": 277247.4457070707, "avg_output_tokens_per_task": 21110.815025252527, "last_run_timestamp": "2026-10-10T16:11:14.296Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8741300360531311, "accuracy_std_dev": 0.0120411700506014, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.008937657811511701, "avg_duration_per_task_ms": 436322.61416824796, "avg_output_tokens_per_task": 21945.78494623656, "last_run_timestamp": "2026-10-10T08:31:34.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.8740979999999999, "accuracy_std_dev": 0.06811118283890162, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.127235829004329, "avg_duration_per_task_ms": 1398882.1861471862, "avg_output_tokens_per_task": 55498.13997113997, "last_run_timestamp": "2026-10-10T16:19:40.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8738600668693008, "accuracy_std_dev": 0.0037938444143111693, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.044807096251266464, "avg_duration_per_task_ms": 186049.48733535968, "avg_output_tokens_per_task": 38929.8591691996, "last_run_timestamp": "2026-10-10T08:16:44.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008290222222222223, "avg_duration_per_task_ms": 529447, "avg_output_tokens_per_task": 29241.171717171717, "last_run_timestamp": "2026-07-18T08:31:23.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8736602893401014, "accuracy_std_dev": 0.018240759785588822, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.033190517766497465, "avg_duration_per_task_ms": 731117.2707275804, "avg_output_tokens_per_task": 38434.73942470389, "last_run_timestamp": "2026-10-03T11:12:58.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8733163749999999, "accuracy_std_dev": 0.015051805696805129, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025130481060606065, "avg_duration_per_task_ms": 723244.2184343435, "avg_output_tokens_per_task": 35149.60164141414, "last_run_timestamp": "2026-10-10T16:32:56.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.873140968503937, "accuracy_std_dev": 0.008141500000000024, "run_count": 2, "total_questions": 381, "avg_cost_per_task_usd": 0.10504513910761155, "avg_duration_per_task_ms": 1003143.2047244095, "avg_output_tokens_per_task": 42008.77427821522, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8727272475524476, "accuracy_std_dev": 0.029233686851601524, "run_count": 8, "total_questions": 1430, "avg_cost_per_task_usd": 0.09700304615384615, "avg_duration_per_task_ms": 194874.37272727274, "avg_output_tokens_per_task": 26397.99300699301, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8727271395348837, "accuracy_std_dev": 0.05132840041585165, "run_count": 12, "total_questions": 2365, "avg_cost_per_task_usd": 0.07667611374207188, "avg_duration_per_task_ms": 546814.6951374207, "avg_output_tokens_per_task": 38662.21902748414, "last_run_timestamp": "2026-10-10T08:21:47.281Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8724745, "accuracy_std_dev": 0.008858340547190563, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008325782828282828, "avg_duration_per_task_ms": 77307.88005050505, "avg_output_tokens_per_task": 7842.704545454545, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8723906000000001, "accuracy_std_dev": 0.017443444896006063, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.032869219191919194, "avg_duration_per_task_ms": 141993.59494949496, "avg_output_tokens_per_task": 24250.128282828282, "last_run_timestamp": "2026-10-10T08:24:43.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8723343333333334, "accuracy_std_dev": 0.0042828368194716806, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0043181245791245795, "avg_duration_per_task_ms": 40010.954545454544, "avg_output_tokens_per_task": 7867.228956228956, "last_run_timestamp": "2026-10-10T08:23:23.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8721631051412692, "accuracy_std_dev": 0.00936942620732698, "run_count": 11, "total_questions": 2159, "avg_cost_per_task_usd": 0.0081905757295044, "avg_duration_per_task_ms": 336770.76748494676, "avg_output_tokens_per_task": 28956.01343214451, "last_run_timestamp": "2026-10-10T08:22:41.871Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.8720540000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04668046464646465, "avg_duration_per_task_ms": 217379.2803030303, "avg_output_tokens_per_task": 38760.13888888889, "last_run_timestamp": "2026-10-10T08:22:24.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8717762017353579, "accuracy_std_dev": 0.02072198720771574, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.1588113485177151, "avg_duration_per_task_ms": 951795.9934924078, "avg_output_tokens_per_task": 53171.428054953, "last_run_timestamp": "2026-10-10T16:20:06.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8716425648854962, "accuracy_std_dev": 0.013438421323454144, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.1836925326547922, "avg_duration_per_task_ms": 775815.9338422392, "avg_output_tokens_per_task": 53092.24766751484, "last_run_timestamp": "2026-10-10T16:26:47.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8714859264426126, "accuracy_std_dev": 0.00752103815972237, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.06945589727330374, "avg_duration_per_task_ms": 276858.71528218134, "avg_output_tokens_per_task": 11308.629676601142, "last_run_timestamp": "2026-10-10T08:22:14.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8714225, "accuracy_std_dev": 0.004885840587862051, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06337124747474747, "avg_duration_per_task_ms": 284598.4861111111, "avg_output_tokens_per_task": 21034.015151515152, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8714174896551725, "accuracy_std_dev": 0.09994490478357737, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.0273483167816092, "avg_duration_per_task_ms": 210323.05977011495, "avg_output_tokens_per_task": 23588.830344827587, "last_run_timestamp": "2026-10-10T16:22:50.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.8713650909090909, "accuracy_std_dev": 0.010760647289203607, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.006425433425160699, "avg_duration_per_task_ms": 272145.9591368228, "avg_output_tokens_per_task": 18480.175849403124, "last_run_timestamp": "2026-10-10T08:22:19.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.871212, "accuracy_std_dev": 0.007576000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00393519191919192, "avg_duration_per_task_ms": 37885.9494949495, "avg_output_tokens_per_task": 7630.535353535353, "last_run_timestamp": "2026-10-10T08:33:36.051Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8712119166666668, "accuracy_std_dev": 0.01635831591504423, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.008297640993265992, "avg_duration_per_task_ms": 229029.07449494948, "avg_output_tokens_per_task": 29392.178030303032, "last_run_timestamp": "2026-10-10T08:22:41.239Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.8711735553145336, "accuracy_std_dev": 0.01949786803730092, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.05907025379609545, "avg_duration_per_task_ms": 512258.996384671, "avg_output_tokens_per_task": 33994.65726681128, "last_run_timestamp": "2026-10-10T16:29:14.779Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8706108571428572, "accuracy_std_dev": 0.005578448284720969, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008177433621933622, "avg_duration_per_task_ms": 408394.5281385281, "avg_output_tokens_per_task": 28879.512265512265, "last_run_timestamp": "2026-10-10T16:19:50.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8703705, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004056603535353536, "avg_duration_per_task_ms": 39122.805555555555, "avg_output_tokens_per_task": 7875.414141414141, "last_run_timestamp": "2026-10-10T08:24:29.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.8701297142857144, "accuracy_std_dev": 0.009751370859148623, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0770424466089466, "avg_duration_per_task_ms": 106865.99062049061, "avg_output_tokens_per_task": 12502.755411255412, "last_run_timestamp": "2026-10-10T08:24:11.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8698495694200352, "accuracy_std_dev": 0.013406448590229597, "run_count": 9, "total_questions": 1707, "avg_cost_per_task_usd": 0.00814071236086702, "avg_duration_per_task_ms": 243885.67662565905, "avg_output_tokens_per_task": 29621.29173989455, "last_run_timestamp": "2026-10-10T16:20:12.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8694081428571427, "accuracy_std_dev": 0.012000957395481546, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0817209256854257, "avg_duration_per_task_ms": 202473.6810966811, "avg_output_tokens_per_task": 13277.033189033189, "last_run_timestamp": "2026-10-10T08:13:06.615Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8693955131578947, "accuracy_std_dev": 0.019648291273268125, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.07663164254385965, "avg_duration_per_task_ms": 433978.8881578947, "avg_output_tokens_per_task": 23705.32821637427, "last_run_timestamp": "2026-10-08T23:14:17.356Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8693361387478848, "accuracy_std_dev": 0.024099081962433728, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.08116604963338973, "avg_duration_per_task_ms": 411609.69035532995, "avg_output_tokens_per_task": 32356.587704455724, "last_run_timestamp": "2026-10-10T16:22:22.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8692479999999999, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22171807575757574, "avg_duration_per_task_ms": 260359.50673400675, "avg_output_tokens_per_task": 22076.11111111111, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8691678571428573, "accuracy_std_dev": 0.022635118532483796, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15808130735930737, "avg_duration_per_task_ms": 1043145.4696969697, "avg_output_tokens_per_task": 60299.43795093795, "last_run_timestamp": "2026-10-10T16:27:14.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8687731366353544, "accuracy_std_dev": 0.028735792378518143, "run_count": 13, "total_questions": 2342, "avg_cost_per_task_usd": 0.011254341588385995, "avg_duration_per_task_ms": 345256.6985482494, "avg_output_tokens_per_task": 20267.622117847994, "last_run_timestamp": "2026-10-10T08:22:27.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13888562626262627, "avg_duration_per_task_ms": 1028019.2020202021, "avg_output_tokens_per_task": 46184.969696969696, "last_run_timestamp": "2026-07-18T16:16:27.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8682058571428571, "accuracy_std_dev": 0.007673379706651383, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07688844300144301, "avg_duration_per_task_ms": 83582.13492063493, "avg_output_tokens_per_task": 12574.14357864358, "last_run_timestamp": "2026-10-10T08:21:46.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8681256666666667, "accuracy_std_dev": 0.00955634355923971, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004278478114478115, "avg_duration_per_task_ms": 35528.6632996633, "avg_output_tokens_per_task": 7802.548821548821, "last_run_timestamp": "2026-10-10T08:40:38.947Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8680553750000001, "accuracy_std_dev": 0.004605583891796467, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07070611994949497, "avg_duration_per_task_ms": 232289.42803030304, "avg_output_tokens_per_task": 11517.214015151516, "last_run_timestamp": "2026-10-10T08:22:11.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8678449999999999, "accuracy_std_dev": 0.007760578667212899, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0038241401515151513, "avg_duration_per_task_ms": 408271.3705808081, "avg_output_tokens_per_task": 29802.972222222223, "last_run_timestamp": "2026-10-10T16:19:50.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8676699264214046, "accuracy_std_dev": 0.009490809964264255, "run_count": 9, "total_questions": 1495, "avg_cost_per_task_usd": 0.05216587558528428, "avg_duration_per_task_ms": 421713.7277591973, "avg_output_tokens_per_task": 17294.008695652174, "last_run_timestamp": "2026-10-10T16:20:24.763Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.867634625, "accuracy_std_dev": 0.007431114989311851, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025694838383838382, "avg_duration_per_task_ms": 248942.36553030304, "avg_output_tokens_per_task": 21167.008207070707, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.867003, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11555696464646466, "avg_duration_per_task_ms": 292702.6111111111, "avg_output_tokens_per_task": 29334.075757575756, "last_run_timestamp": "2026-07-18T08:27:05.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.8666407869362363, "accuracy_std_dev": 0.0883070185402466, "run_count": 13, "total_questions": 2572, "avg_cost_per_task_usd": 0.26275894012441675, "avg_duration_per_task_ms": 494855.53265940904, "avg_output_tokens_per_task": 63072.154743390354, "last_run_timestamp": "2026-10-10T08:36:25.212Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8665821595946803, "accuracy_std_dev": 0.017909527324426373, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.006400538315389487, "avg_duration_per_task_ms": 337664.5319822673, "avg_output_tokens_per_task": 21313.250158328057, "last_run_timestamp": "2026-10-10T08:29:42.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8662817142857142, "accuracy_std_dev": 0.018765141853327668, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.19692205699855703, "avg_duration_per_task_ms": 428251.06204906205, "avg_output_tokens_per_task": 57606.430014430014, "last_run_timestamp": "2026-10-10T16:23:16.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.8661615, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.030201459595959598, "avg_duration_per_task_ms": 496509.8611111111, "avg_output_tokens_per_task": 34831.292929292926, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8659881251908396, "accuracy_std_dev": 0.03841903799472341, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.09208694758269721, "avg_duration_per_task_ms": 534219.7852417303, "avg_output_tokens_per_task": 23722.84885496183, "last_run_timestamp": "2026-10-10T08:26:20.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.86574075, "accuracy_std_dev": 0.004187816696979481, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00415591287878788, "avg_duration_per_task_ms": 35585.20075757576, "avg_output_tokens_per_task": 7676.222222222223, "last_run_timestamp": "2026-10-10T08:24:11.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.865689794871795, "accuracy_std_dev": 0.004877364201651907, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.005803672161172161, "avg_duration_per_task_ms": 153675.5, "avg_output_tokens_per_task": 20639.26923076923, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.865530125, "accuracy_std_dev": 0.009082931319754368, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07008002651515152, "avg_duration_per_task_ms": 228487.37373737374, "avg_output_tokens_per_task": 11410.660353535353, "last_run_timestamp": "2026-10-10T08:31:34.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8651996980273142, "accuracy_std_dev": 0.03307372620434535, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.13402964896307534, "avg_duration_per_task_ms": 611597.9382903388, "avg_output_tokens_per_task": 40351.92817400101, "last_run_timestamp": "2026-10-10T08:14:39.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8650791428571427, "accuracy_std_dev": 0.02076397686536241, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11897097835497836, "avg_duration_per_task_ms": 585047.1832611832, "avg_output_tokens_per_task": 35346.81024531025, "last_run_timestamp": "2026-10-10T16:21:22.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.8649788658227848, "accuracy_std_dev": 0.00034200000000000896, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.5347964379746836, "avg_duration_per_task_ms": 1600008.7797468354, "avg_output_tokens_per_task": 61634.33417721519, "last_run_timestamp": "2026-10-10T16:21:45.788Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8648158710912572, "accuracy_std_dev": 0.014610716473170469, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.07521307466496491, "avg_duration_per_task_ms": 273268.63114231016, "avg_output_tokens_per_task": 20767.77855775367, "last_run_timestamp": "2026-10-10T16:19:34.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8646463, "accuracy_std_dev": 0.01622113306184252, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.08817965, "avg_duration_per_task_ms": 193592.76363636364, "avg_output_tokens_per_task": 24362.06313131313, "last_run_timestamp": "2026-10-10T16:11:14.298Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8645964220532318, "accuracy_std_dev": 0.028631942149459588, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.10253569771863118, "avg_duration_per_task_ms": 364033.2845373891, "avg_output_tokens_per_task": 23065.25855513308, "last_run_timestamp": "2026-10-10T08:22:13.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8643883344182262, "accuracy_std_dev": 0.01774344793127301, "run_count": 7, "total_questions": 1229, "avg_cost_per_task_usd": 0.0051325646867371845, "avg_duration_per_task_ms": 651023.4035801465, "avg_output_tokens_per_task": 28234.647681041497, "last_run_timestamp": "2026-10-08T23:29:09.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8643575714285714, "accuracy_std_dev": 0.014782409815695945, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.042783132756132755, "avg_duration_per_task_ms": 500704.0137085137, "avg_output_tokens_per_task": 34254.27994227994, "last_run_timestamp": "2026-10-10T16:29:43.875Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8642674999999999, "accuracy_std_dev": 0.01013372239110584, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008602727904040404, "avg_duration_per_task_ms": 273218.547979798, "avg_output_tokens_per_task": 30406.29861111111, "last_run_timestamp": "2026-10-10T16:10:48.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.86405725, "accuracy_std_dev": 0.017018053235828706, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07367723863636363, "avg_duration_per_task_ms": 277870.3768939394, "avg_output_tokens_per_task": 20967.248106060608, "last_run_timestamp": "2026-10-10T16:19:34.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.863779637554585, "accuracy_std_dev": 0.014583686901214414, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.13604484788937413, "avg_duration_per_task_ms": 292893.29330422124, "avg_output_tokens_per_task": 27835.631004366813, "last_run_timestamp": "2026-10-10T08:22:50.053Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.86363625, "accuracy_std_dev": 0.015612150218259494, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.049225773358585856, "avg_duration_per_task_ms": 723327.8787878788, "avg_output_tokens_per_task": 33961.57638888889, "last_run_timestamp": "2026-10-10T16:21:33.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008496712121212122, "avg_duration_per_task_ms": 739364.4242424242, "avg_output_tokens_per_task": 30053.868686868685, "last_run_timestamp": "2026-07-18T08:30:06.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.863636, "accuracy_std_dev": 0.007273551310513057, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020974939393939393, "avg_duration_per_task_ms": 363755.6632996633, "avg_output_tokens_per_task": 34831.013468013465, "last_run_timestamp": "2026-10-03T11:53:46.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.863636, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22125699158249157, "avg_duration_per_task_ms": 259075.24915824915, "avg_output_tokens_per_task": 22030.511784511786, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8632620000000001, "accuracy_std_dev": 0.0411666386369675, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17732408585858583, "avg_duration_per_task_ms": 45341.72558922559, "avg_output_tokens_per_task": 3345.6531986531986, "last_run_timestamp": "2026-10-10T08:22:26.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8628614010378057, "accuracy_std_dev": 0.04100774563321383, "run_count": 7, "total_questions": 1349, "avg_cost_per_task_usd": 0.015537737583395106, "avg_duration_per_task_ms": 567965.6375092661, "avg_output_tokens_per_task": 38495.872498146775, "last_run_timestamp": "2026-10-10T08:29:25.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.8625952061068702, "accuracy_std_dev": 0.005736999999999992, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.043527526717557254, "avg_duration_per_task_ms": 365946.2213740458, "avg_output_tokens_per_task": 24956.239185750637, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8625138888888889, "accuracy_std_dev": 0.02716878807440808, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00797921548821549, "avg_duration_per_task_ms": 504816.8608305275, "avg_output_tokens_per_task": 25080.10830527497, "last_run_timestamp": "2026-10-03T10:11:44.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8624318766716196, "accuracy_std_dev": 0.015635397712647187, "run_count": 7, "total_questions": 1346, "avg_cost_per_task_usd": 0.08403349479940564, "avg_duration_per_task_ms": 364844.5950965825, "avg_output_tokens_per_task": 25975.783060921247, "last_run_timestamp": "2026-10-10T08:23:49.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.862290694980695, "accuracy_std_dev": 0.013680701809483308, "run_count": 4, "total_questions": 777, "avg_cost_per_task_usd": 0.02531409137709138, "avg_duration_per_task_ms": 145459.53539253538, "avg_output_tokens_per_task": 38737.666666666664, "last_run_timestamp": "2026-10-10T08:21:21.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8622895, "accuracy_std_dev": 0.009153046839714083, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.8048450656565656, "avg_duration_per_task_ms": 460341.97323232325, "avg_output_tokens_per_task": 32021.969191919194, "last_run_timestamp": "2026-10-10T08:14:45.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.8621211000000002, "accuracy_std_dev": 0.014333635361972893, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.7483498358585858, "avg_duration_per_task_ms": 493892.65252525255, "avg_output_tokens_per_task": 29772.12373737374, "last_run_timestamp": "2026-10-10T08:21:50.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004047368686868687, "avg_duration_per_task_ms": 38582.818181818184, "avg_output_tokens_per_task": 7859.823232323232, "last_run_timestamp": "2026-10-10T08:13:08.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.8619530000000001, "accuracy_std_dev": 0.009912179612308596, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.07266585185185186, "avg_duration_per_task_ms": 469919.06565656565, "avg_output_tokens_per_task": 56748.079124579126, "last_run_timestamp": "2026-10-10T16:19:34.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8614478, "accuracy_std_dev": 0.008749266401247596, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.8440670252525252, "avg_duration_per_task_ms": 414934.17070707073, "avg_output_tokens_per_task": 33571.59191919192, "last_run_timestamp": "2026-10-10T08:22:13.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8614377764705882, "accuracy_std_dev": 0.012431326538225896, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.051462403921568624, "avg_duration_per_task_ms": 413319.64183006535, "avg_output_tokens_per_task": 26795.732026143793, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8612312857142858, "accuracy_std_dev": 0.008191248094578664, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005079496392496392, "avg_duration_per_task_ms": 242062.32467532466, "avg_output_tokens_per_task": 28953.08080808081, "last_run_timestamp": "2026-10-10T16:20:24.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.861142954, "accuracy_std_dev": 0.038697731064696234, "run_count": 18, "total_questions": 3500, "avg_cost_per_task_usd": 0.004615875999999999, "avg_duration_per_task_ms": 95064.51371428571, "avg_output_tokens_per_task": 17377.474, "last_run_timestamp": "2026-10-10T08:28:15.461Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8609939949452401, "accuracy_std_dev": 0.034291003424934796, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.12772796882898063, "avg_duration_per_task_ms": 665372.3294018534, "avg_output_tokens_per_task": 42817.63689974726, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8609694285714284, "accuracy_std_dev": 0.023293052022008633, "run_count": 4, "total_questions": 784, "avg_cost_per_task_usd": 0.10234140178571428, "avg_duration_per_task_ms": 463624.993622449, "avg_output_tokens_per_task": 30066.539540816328, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8608923543307087, "accuracy_std_dev": 0.017439499999999997, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.003837814960629921, "avg_duration_per_task_ms": 298144.1771653543, "avg_output_tokens_per_task": 11523.807086614173, "last_run_timestamp": "2026-10-03T17:04:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.86077435, "accuracy_std_dev": 0.007771512898239295, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.8830627252525252, "avg_duration_per_task_ms": 416252.63737373735, "avg_output_tokens_per_task": 33448.936111111114, "last_run_timestamp": "2026-10-10T08:21:47.547Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8606576691394658, "accuracy_std_dev": 0.011001380598888357, "run_count": 24, "total_questions": 4044, "avg_cost_per_task_usd": 0.16706140232443123, "avg_duration_per_task_ms": 253403.07789317507, "avg_output_tokens_per_task": 30580.46612265084, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8605299818688982, "accuracy_std_dev": 0.04545858647428449, "run_count": 5, "total_questions": 717, "avg_cost_per_task_usd": 0.016286825662482563, "avg_duration_per_task_ms": 469492.70013947, "avg_output_tokens_per_task": 32621.504881450488, "last_run_timestamp": "2026-10-10T08:14:43.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8604376000000001, "accuracy_std_dev": 0.018325598255991542, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0985075101010101, "avg_duration_per_task_ms": 504799.2101010101, "avg_output_tokens_per_task": 26512.831818181818, "last_run_timestamp": "2026-10-10T08:14:42.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8604375000000001, "accuracy_std_dev": 0.010417175387311071, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.8523096944444445, "avg_duration_per_task_ms": 482090.7853535354, "avg_output_tokens_per_task": 33901.62272727273, "last_run_timestamp": "2026-10-10T08:22:14.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8604060246555475, "accuracy_std_dev": 0.014343180030514657, "run_count": 7, "total_questions": 1379, "avg_cost_per_task_usd": 0.07309712690355329, "avg_duration_per_task_ms": 193422.33792603336, "avg_output_tokens_per_task": 24090.86439448876, "last_run_timestamp": "2026-10-10T08:24:32.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8603742847113885, "accuracy_std_dev": 0.024463459108454412, "run_count": 7, "total_questions": 1282, "avg_cost_per_task_usd": 0.02198177067082683, "avg_duration_per_task_ms": 379545.007800312, "avg_output_tokens_per_task": 23938.183307332292, "last_run_timestamp": "2026-10-10T16:11:11.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8602841643713905, "accuracy_std_dev": 0.03073868173578963, "run_count": 12, "total_questions": 2251, "avg_cost_per_task_usd": 0.025296473567303423, "avg_duration_per_task_ms": 476437.5064415815, "avg_output_tokens_per_task": 24104.674811195026, "last_run_timestamp": "2026-10-10T08:22:28.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.859932032938258, "accuracy_std_dev": 0.020959658153797075, "run_count": 37, "total_questions": 7256, "avg_cost_per_task_usd": 0.26751970507166484, "avg_duration_per_task_ms": 334415.0935777288, "avg_output_tokens_per_task": 48371.618936052924, "last_run_timestamp": "2026-10-10T08:27:36.984Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.8599289999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 188, "avg_cost_per_task_usd": 0.023036143617021276, "avg_duration_per_task_ms": 1599986.2287234042, "avg_output_tokens_per_task": 29232.622340425532, "last_run_timestamp": "2026-07-18T08:57:54.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8598485, "accuracy_std_dev": 0.007374290067525129, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.06667829040404041, "avg_duration_per_task_ms": 218736.0157828283, "avg_output_tokens_per_task": 10845.020202020201, "last_run_timestamp": "2026-10-10T08:14:46.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8597080000000001, "accuracy_std_dev": 0.011568646146661513, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.843727255892256, "avg_duration_per_task_ms": 453475.51683501684, "avg_output_tokens_per_task": 33571.67003367003, "last_run_timestamp": "2026-10-10T08:30:17.849Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8587461904761905, "accuracy_std_dev": 0.026229751534903395, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.15894341366041365, "avg_duration_per_task_ms": 153104.68903318903, "avg_output_tokens_per_task": 15468.237133237133, "last_run_timestamp": "2026-10-10T08:22:11.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2511099898989899, "avg_duration_per_task_ms": 290522.3080808081, "avg_output_tokens_per_task": 62590.06565656565, "last_run_timestamp": "2026-07-18T08:26:40.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8585856363636363, "accuracy_std_dev": 0.036178356787833384, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.18056699632690543, "avg_duration_per_task_ms": 44952.12350780533, "avg_output_tokens_per_task": 3404.9237832874196, "last_run_timestamp": "2026-10-10T08:14:48.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8579736363636363, "accuracy_std_dev": 0.04127002151082713, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1750287943067034, "avg_duration_per_task_ms": 43575.79752066116, "avg_output_tokens_per_task": 3294.965564738292, "last_run_timestamp": "2026-10-10T08:14:22.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8577798183779121, "accuracy_std_dev": 0.020787162919696366, "run_count": 12, "total_questions": 2318, "avg_cost_per_task_usd": 0.03407388869715272, "avg_duration_per_task_ms": 74054.28817946506, "avg_output_tokens_per_task": 23021.116048317515, "last_run_timestamp": "2026-10-10T08:38:05.766Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8573280311284046, "accuracy_std_dev": 0.008776662705721345, "run_count": 4, "total_questions": 771, "avg_cost_per_task_usd": 0.11767329182879378, "avg_duration_per_task_ms": 355412.9636835279, "avg_output_tokens_per_task": 29743.00907911803, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8569023750000001, "accuracy_std_dev": 0.057848586717692385, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10924076767676767, "avg_duration_per_task_ms": 469852.4330808081, "avg_output_tokens_per_task": 27711.996843434343, "last_run_timestamp": "2026-10-10T08:22:14.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "WandB", "accuracy": 0.8569019999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006055888888888888, "avg_duration_per_task_ms": 490894.14646464644, "avg_output_tokens_per_task": 17030.60606060606, "last_run_timestamp": "2026-07-18T08:04:02.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8568846555023923, "accuracy_std_dev": 0.008223840494828117, "run_count": 16, "total_questions": 3135, "avg_cost_per_task_usd": 0.06289737799043063, "avg_duration_per_task_ms": 600548.7228070175, "avg_output_tokens_per_task": 34655.311323763955, "last_run_timestamp": "2026-10-10T16:26:11.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8565061764705884, "accuracy_std_dev": 0.012525463463235917, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.13928274509803923, "avg_duration_per_task_ms": 437654.1420083185, "avg_output_tokens_per_task": 42837.04515745692, "last_run_timestamp": "2026-10-10T08:21:41.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8563971000000001, "accuracy_std_dev": 0.01879277008559409, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.7556142424242424, "avg_duration_per_task_ms": 501990.4429292929, "avg_output_tokens_per_task": 30104.112626262628, "last_run_timestamp": "2026-10-10T08:30:49.776Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.8560605, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02547615909090909, "avg_duration_per_task_ms": 266390.93434343435, "avg_output_tokens_per_task": 21055.325757575756, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8557689653846154, "accuracy_std_dev": 0.016989351495046, "run_count": 3, "total_questions": 520, "avg_cost_per_task_usd": 0.06363424807692308, "avg_duration_per_task_ms": 459711.01153846155, "avg_output_tokens_per_task": 23017.521153846155, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8557238, "accuracy_std_dev": 0.018226201572461534, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.8383144191919192, "avg_duration_per_task_ms": 453248.6974747475, "avg_output_tokens_per_task": 33360.03333333333, "last_run_timestamp": "2026-10-10T08:22:13.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8555989032258063, "accuracy_std_dev": 0.02602705082664132, "run_count": 31, "total_questions": 6138, "avg_cost_per_task_usd": 0.17111695454545456, "avg_duration_per_task_ms": 157950.86656891496, "avg_output_tokens_per_task": 15997.16568914956, "last_run_timestamp": "2026-10-10T08:40:15.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8554993333333333, "accuracy_std_dev": 0.02849418848740135, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12659392845117845, "avg_duration_per_task_ms": 524253.6026936027, "avg_output_tokens_per_task": 33349.10774410774, "last_run_timestamp": "2026-10-10T16:20:22.055Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8554712499999998, "accuracy_std_dev": 0.018079248695327472, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.8025045977272727, "avg_duration_per_task_ms": 479096.1358585859, "avg_output_tokens_per_task": 30416.972222222223, "last_run_timestamp": "2026-10-10T08:24:12.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8552682396551725, "accuracy_std_dev": 0.0376233031656357, "run_count": 30, "total_questions": 5220, "avg_cost_per_task_usd": 0.013701367241379304, "avg_duration_per_task_ms": 406055.5988505747, "avg_output_tokens_per_task": 16990.452490421456, "last_run_timestamp": "2026-10-10T08:22:25.039Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.855219, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.19856715151515153, "avg_duration_per_task_ms": 638184.691919192, "avg_output_tokens_per_task": 58287.33838383838, "last_run_timestamp": "2026-07-18T08:26:48.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8550479847715735, "accuracy_std_dev": 0.00436888279642389, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.24340484940778345, "avg_duration_per_task_ms": 425015.06429780036, "avg_output_tokens_per_task": 54319.2165820643, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8550316666666666, "accuracy_std_dev": 0.029012228253311092, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8331611588103255, "avg_duration_per_task_ms": 427437.11616161617, "avg_output_tokens_per_task": 33239.09371492705, "last_run_timestamp": "2026-10-10T08:21:42.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.854800172815534, "accuracy_std_dev": 0.012641847777693532, "run_count": 8, "total_questions": 1545, "avg_cost_per_task_usd": 0.1002615708737864, "avg_duration_per_task_ms": 668327.793527508, "avg_output_tokens_per_task": 36663.679611650485, "last_run_timestamp": "2026-09-19T08:55:10.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.854592, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.08009020408163266, "avg_duration_per_task_ms": 1565820.9795918367, "avg_output_tokens_per_task": 40916.48469387755, "last_run_timestamp": "2026-07-18T16:22:39.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8545453999999999, "accuracy_std_dev": 0.008010288199559339, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006016572727272727, "avg_duration_per_task_ms": 589898.7232323232, "avg_output_tokens_per_task": 28997.2, "last_run_timestamp": "2026-10-03T17:45:03.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8543769166666666, "accuracy_std_dev": 0.015710205910269136, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7564824368686868, "avg_duration_per_task_ms": 502571.60101010103, "avg_output_tokens_per_task": 30160.646464646463, "last_run_timestamp": "2026-10-10T08:22:14.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8543189326500733, "accuracy_std_dev": 0.02429014124777243, "run_count": 7, "total_questions": 1366, "avg_cost_per_task_usd": 0.13498622767203514, "avg_duration_per_task_ms": 368358.08931185945, "avg_output_tokens_per_task": 30688.598828696926, "last_run_timestamp": "2026-10-10T08:25:24.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8542019060913706, "accuracy_std_dev": 0.006301576321753829, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.05433487690355329, "avg_duration_per_task_ms": 423724.30245346867, "avg_output_tokens_per_task": 27804.93189509306, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8540081848101264, "accuracy_std_dev": 0.01478060943056438, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.10410900675105485, "avg_duration_per_task_ms": 306238.13670886075, "avg_output_tokens_per_task": 29563.783966244726, "last_run_timestamp": "2026-10-10T08:34:20.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8539943636363636, "accuracy_std_dev": 0.039917099808734806, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.17991296786042243, "avg_duration_per_task_ms": 45861.22451790634, "avg_output_tokens_per_task": 3391.032598714417, "last_run_timestamp": "2026-10-10T08:28:02.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.8539559999999999, "accuracy_std_dev": 0.062180043611274514, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07175601767676767, "avg_duration_per_task_ms": 649147.8876262626, "avg_output_tokens_per_task": 27046.439393939392, "last_run_timestamp": "2026-10-10T16:30:18.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8537527991169976, "accuracy_std_dev": 0.017478527754711547, "run_count": 6, "total_questions": 906, "avg_cost_per_task_usd": 0.0038477715231788083, "avg_duration_per_task_ms": 380553.4061810154, "avg_output_tokens_per_task": 8747.8233995585, "last_run_timestamp": "2026-10-10T08:22:09.526Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.853535090909091, "accuracy_std_dev": 0.02817953127377952, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.15646832736455463, "avg_duration_per_task_ms": 152222.10376492195, "avg_output_tokens_per_task": 15431.129476584021, "last_run_timestamp": "2026-10-10T08:22:41.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.853478567104149, "accuracy_std_dev": 0.016412097348675454, "run_count": 25, "total_questions": 4724, "avg_cost_per_task_usd": 0.16086846613039799, "avg_duration_per_task_ms": 197709.6486028789, "avg_output_tokens_per_task": 28618.75550381033, "last_run_timestamp": "2026-10-10T08:24:01.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8534097609942638, "accuracy_std_dev": 0.02277588037678235, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.0424615430210325, "avg_duration_per_task_ms": 655970.3365200765, "avg_output_tokens_per_task": 35467.44550669216, "last_run_timestamp": "2026-10-10T16:20:00.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8530642646239555, "accuracy_std_dev": 0.022329477337983097, "run_count": 4, "total_questions": 718, "avg_cost_per_task_usd": 0.02860370194986072, "avg_duration_per_task_ms": 460751.2409470752, "avg_output_tokens_per_task": 34791.85933147632, "last_run_timestamp": "2026-10-10T08:22:41.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8529741666666667, "accuracy_std_dev": 0.015541427019600089, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11713632154882155, "avg_duration_per_task_ms": 720822.8728956229, "avg_output_tokens_per_task": 37315.70791245791, "last_run_timestamp": "2026-10-10T16:20:03.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8523328571428571, "accuracy_std_dev": 0.014361701465341489, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10223797113997113, "avg_duration_per_task_ms": 552415.2619047619, "avg_output_tokens_per_task": 29613.10101010101, "last_run_timestamp": "2026-10-10T08:22:27.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8523109090909091, "accuracy_std_dev": 0.012141784887917529, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0287087217630854, "avg_duration_per_task_ms": 734317.4224058769, "avg_output_tokens_per_task": 32736.72819100092, "last_run_timestamp": "2026-10-10T08:34:24.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 144, "avg_cost_per_task_usd": 0.15792222916666668, "avg_duration_per_task_ms": 798429.2847222222, "avg_output_tokens_per_task": 42935.319444444445, "last_run_timestamp": "2026-07-18T08:29:37.313Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8513466999999999, "accuracy_std_dev": 0.07725069644611626, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007058363131313131, "avg_duration_per_task_ms": 665246.3, "avg_output_tokens_per_task": 29067.647474747475, "last_run_timestamp": "2026-10-10T08:21:25.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8512543790322581, "accuracy_std_dev": 0.05785076520439789, "run_count": 8, "total_questions": 1488, "avg_cost_per_task_usd": 0.010551217069892472, "avg_duration_per_task_ms": 536964.1794354839, "avg_output_tokens_per_task": 20899.05376344086, "last_run_timestamp": "2026-10-10T08:22:13.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8510100909090909, "accuracy_std_dev": 0.009650973153517579, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03987833700642792, "avg_duration_per_task_ms": 393524.6414141414, "avg_output_tokens_per_task": 40088.10973370064, "last_run_timestamp": "2026-10-10T16:21:08.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.8510099999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.26341453282828287, "avg_duration_per_task_ms": 421660.03282828286, "avg_output_tokens_per_task": 59612.60101010101, "last_run_timestamp": "2026-10-10T08:15:00.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8509659760869565, "accuracy_std_dev": 0.01639823188577327, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.07163253550724637, "avg_duration_per_task_ms": 259347.84275362318, "avg_output_tokens_per_task": 18767.492753623188, "last_run_timestamp": "2026-10-10T08:35:07.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.850575, "accuracy_std_dev": null, "run_count": 1, "total_questions": 145, "avg_cost_per_task_usd": 0.015425489655172412, "avg_duration_per_task_ms": 62324.744827586204, "avg_output_tokens_per_task": 15913.096551724138, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.850427, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.014483525641025641, "avg_duration_per_task_ms": 276126.60256410256, "avg_output_tokens_per_task": 12035.96794871795, "last_run_timestamp": "2026-07-18T16:53:38.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8504244789808917, "accuracy_std_dev": 0.0585455720762414, "run_count": 8, "total_questions": 1570, "avg_cost_per_task_usd": 0.0790043025477707, "avg_duration_per_task_ms": 541595.1828025478, "avg_output_tokens_per_task": 21736.728025477707, "last_run_timestamp": "2026-10-10T08:21:42.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8500727423580785, "accuracy_std_dev": 0.017073044892999246, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.10693941775836974, "avg_duration_per_task_ms": 193416.326783115, "avg_output_tokens_per_task": 29417.200145560408, "last_run_timestamp": "2026-10-10T08:26:05.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8496872857142856, "accuracy_std_dev": 0.02328767377645021, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014274932178932179, "avg_duration_per_task_ms": 598242.8015873015, "avg_output_tokens_per_task": 35895.869408369406, "last_run_timestamp": "2026-10-10T08:11:18.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8495985437262357, "accuracy_std_dev": 0.01598636257095404, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.01962292712294043, "avg_duration_per_task_ms": 346902.4087452472, "avg_output_tokens_per_task": 39034.471482889734, "last_run_timestamp": "2026-10-10T08:22:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8494668333333334, "accuracy_std_dev": 0.08246985866245651, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09358024705387205, "avg_duration_per_task_ms": 746941.3282828283, "avg_output_tokens_per_task": 41492.88888888889, "last_run_timestamp": "2026-10-10T08:22:24.596Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8493871292775664, "accuracy_std_dev": 0.028830055092028094, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.012782455006337136, "avg_duration_per_task_ms": 567459.4005069708, "avg_output_tokens_per_task": 36588.9752851711, "last_run_timestamp": "2026-10-10T08:24:22.181Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.849375142857143, "accuracy_std_dev": 0.05373653913902533, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.09692596960486322, "avg_duration_per_task_ms": 658494.2613981763, "avg_output_tokens_per_task": 27042.529888551166, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8493666190476191, "accuracy_std_dev": 0.05184374805167303, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.1855372905242905, "avg_duration_per_task_ms": 40993.36219336219, "avg_output_tokens_per_task": 3341.2967772967772, "last_run_timestamp": "2026-10-10T08:22:13.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.8493151205479452, "accuracy_std_dev": 0.01972059190006447, "run_count": 3, "total_questions": 365, "avg_cost_per_task_usd": 0.06587333698630138, "avg_duration_per_task_ms": 279697.01369863015, "avg_output_tokens_per_task": 16490.610958904108, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8490420331034483, "accuracy_std_dev": 0.010617213932381244, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.030508747126436777, "avg_duration_per_task_ms": 708403.5742528735, "avg_output_tokens_per_task": 31823.873563218393, "last_run_timestamp": "2026-10-10T08:22:26.996Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.848969343283582, "accuracy_std_dev": 0.009530391676106496, "run_count": 12, "total_questions": 2345, "avg_cost_per_task_usd": 0.07047954541577824, "avg_duration_per_task_ms": 213416.39104477613, "avg_output_tokens_per_task": 21924.39616204691, "last_run_timestamp": "2026-10-10T08:14:48.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8487653333333334, "accuracy_std_dev": 0.03509082742762781, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015535906565656565, "avg_duration_per_task_ms": 365861.50505050505, "avg_output_tokens_per_task": 27936.9234006734, "last_run_timestamp": "2026-10-10T16:10:44.131Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8486160259927799, "accuracy_std_dev": 0.03690580233015742, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.16769674440433213, "avg_duration_per_task_ms": 1001771.3018050542, "avg_output_tokens_per_task": 54149.948736462095, "last_run_timestamp": "2026-10-10T16:21:33.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8485233052899935, "accuracy_std_dev": 0.011509616348949264, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.006999760994263861, "avg_duration_per_task_ms": 550728.0395156151, "avg_output_tokens_per_task": 17320.901848311027, "last_run_timestamp": "2026-10-10T16:19:34.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.84850889, "accuracy_std_dev": 0.013594201563902184, "run_count": 10, "total_questions": 1900, "avg_cost_per_task_usd": 0.01998638105263158, "avg_duration_per_task_ms": 280445.96315789473, "avg_output_tokens_per_task": 36076.418947368424, "last_run_timestamp": "2026-10-10T08:23:11.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8484668985163207, "accuracy_std_dev": 0.020724910315372824, "run_count": 9, "total_questions": 1685, "avg_cost_per_task_usd": 0.13222807833827893, "avg_duration_per_task_ms": 291436.95014836796, "avg_output_tokens_per_task": 26711.21543026706, "last_run_timestamp": "2026-10-10T08:28:24.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8484593299406276, "accuracy_std_dev": 0.0127989667296598, "run_count": 7, "total_questions": 1179, "avg_cost_per_task_usd": 0.006712759966072943, "avg_duration_per_task_ms": 475640.88295165397, "avg_output_tokens_per_task": 16540.466497031382, "last_run_timestamp": "2026-10-10T16:22:19.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8484146666666667, "accuracy_std_dev": 0.01925144376115435, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11280877988215487, "avg_duration_per_task_ms": 573583.9625420875, "avg_output_tokens_per_task": 45058.854797979795, "last_run_timestamp": "2026-10-10T08:11:17.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8482337114256125, "accuracy_std_dev": 0.02225972291072427, "run_count": 15, "total_questions": 2897, "avg_cost_per_task_usd": 0.178282096306524, "avg_duration_per_task_ms": 293597.77562996204, "avg_output_tokens_per_task": 43073.98826372109, "last_run_timestamp": "2026-10-10T08:24:31.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8481106666666667, "accuracy_std_dev": 0.036070316075379444, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.7001586167227833, "avg_duration_per_task_ms": 475392.94332211, "avg_output_tokens_per_task": 28044.415263748597, "last_run_timestamp": "2026-10-10T08:22:31.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8477956427969672, "accuracy_std_dev": 0.03319167551259403, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.09577104128053918, "avg_duration_per_task_ms": 1415253.807919124, "avg_output_tokens_per_task": 42907.37742207245, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8477632857142857, "accuracy_std_dev": 0.008091757370740825, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.010911548340548341, "avg_duration_per_task_ms": 434100.7842712843, "avg_output_tokens_per_task": 18528.194083694085, "last_run_timestamp": "2026-10-10T16:21:25.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8476972604562738, "accuracy_std_dev": 0.016172357759461055, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.0376128732572877, "avg_duration_per_task_ms": 613711.562103929, "avg_output_tokens_per_task": 35115.68567807351, "last_run_timestamp": "2026-10-10T16:19:34.452Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8474194046372475, "accuracy_std_dev": 0.028897507729553278, "run_count": 7, "total_questions": 1337, "avg_cost_per_task_usd": 0.1101656335078534, "avg_duration_per_task_ms": 533472.9222139118, "avg_output_tokens_per_task": 33787.17053103964, "last_run_timestamp": "2026-10-10T08:35:43.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8472713558033451, "accuracy_std_dev": 0.026110591357723027, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.00887132336543335, "avg_duration_per_task_ms": 219350.43081601622, "avg_output_tokens_per_task": 13273.467308667005, "last_run_timestamp": "2026-10-10T08:23:49.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8471839090909092, "accuracy_std_dev": 0.009929703716668814, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09975296602387511, "avg_duration_per_task_ms": 580036.968778696, "avg_output_tokens_per_task": 31307.20523415978, "last_run_timestamp": "2026-10-10T16:20:22.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.846689, "accuracy_std_dev": 0.010530862908612949, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.14702261952861953, "avg_duration_per_task_ms": 151760.5090909091, "avg_output_tokens_per_task": 32387.174074074075, "last_run_timestamp": "2026-10-10T08:29:41.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8461861403286979, "accuracy_std_dev": 0.016702053748258686, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.05117073999157185, "avg_duration_per_task_ms": 277147.41508638853, "avg_output_tokens_per_task": 45853.79603876949, "last_run_timestamp": "2026-10-10T16:20:22.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8461756369863013, "accuracy_std_dev": 0.003462455005724478, "run_count": 3, "total_questions": 584, "avg_cost_per_task_usd": 0.07262447602739726, "avg_duration_per_task_ms": 181056.27226027398, "avg_output_tokens_per_task": 31634.909246575342, "last_run_timestamp": "2026-10-03T09:37:27.212Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.846084, "accuracy_std_dev": null, "run_count": 1, "total_questions": 183, "avg_cost_per_task_usd": 0.0059167213114754095, "avg_duration_per_task_ms": 533061.174863388, "avg_output_tokens_per_task": 16599.48087431694, "last_run_timestamp": "2026-10-10T08:22:47.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8459596249999999, "accuracy_std_dev": 0.02667178620254698, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07075826704545456, "avg_duration_per_task_ms": 553373.509469697, "avg_output_tokens_per_task": 22211.534090909092, "last_run_timestamp": "2026-10-10T16:10:30.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.845959375, "accuracy_std_dev": 0.012876192798509, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10514613573232325, "avg_duration_per_task_ms": 332982.2455808081, "avg_output_tokens_per_task": 29278.01388888889, "last_run_timestamp": "2026-10-10T08:14:49.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8456079554432706, "accuracy_std_dev": 0.03356297487340206, "run_count": 33, "total_questions": 6531, "avg_cost_per_task_usd": 0.16561795467769108, "avg_duration_per_task_ms": 150136.5763282805, "avg_output_tokens_per_task": 15309.536211912418, "last_run_timestamp": "2026-10-10T08:28:02.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8455446459920203, "accuracy_std_dev": 0.010937911610913816, "run_count": 14, "total_questions": 2757, "avg_cost_per_task_usd": 0.12664673558215453, "avg_duration_per_task_ms": 332593.6597751179, "avg_output_tokens_per_task": 33927.599927457384, "last_run_timestamp": "2026-10-10T08:26:05.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8450010520231215, "accuracy_std_dev": 0.013556052895639447, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.0843938554913295, "avg_duration_per_task_ms": 983145.6628131021, "avg_output_tokens_per_task": 42748.55427103404, "last_run_timestamp": "2026-10-10T16:20:24.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8449914928670043, "accuracy_std_dev": 0.014523064670429856, "run_count": 11, "total_questions": 2173, "avg_cost_per_task_usd": 0.0884558996778647, "avg_duration_per_task_ms": 407622.08145421074, "avg_output_tokens_per_task": 21997.700414173953, "last_run_timestamp": "2026-10-10T08:27:34.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8448371666666666, "accuracy_std_dev": 0.007861279696857409, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1419990378787879, "avg_duration_per_task_ms": 155301.35437710438, "avg_output_tokens_per_task": 32100.082491582492, "last_run_timestamp": "2026-10-10T08:14:47.788Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8447187666034155, "accuracy_std_dev": 0.033801436872417136, "run_count": 5, "total_questions": 527, "avg_cost_per_task_usd": 0.10369237191650856, "avg_duration_per_task_ms": 295443.4819734345, "avg_output_tokens_per_task": 23792.745730550283, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8445006460176991, "accuracy_std_dev": 0.03377447178960267, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.05367159629161399, "avg_duration_per_task_ms": 217527.86177833966, "avg_output_tokens_per_task": 22261.738727349348, "last_run_timestamp": "2026-10-10T08:22:49.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8444865, "accuracy_std_dev": 0.01261755690496381, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.019410517676767675, "avg_duration_per_task_ms": 420835.0101010101, "avg_output_tokens_per_task": 38594.204545454544, "last_run_timestamp": "2026-10-10T08:22:11.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8444072953657173, "accuracy_std_dev": 0.011477740217046216, "run_count": 10, "total_questions": 1791, "avg_cost_per_task_usd": 0.015475027359017308, "avg_duration_per_task_ms": 603617.405360134, "avg_output_tokens_per_task": 16554.757118927973, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8436994003795065, "accuracy_std_dev": 0.021313050412206686, "run_count": 24, "total_questions": 4743, "avg_cost_per_task_usd": 0.2397977305502846, "avg_duration_per_task_ms": 521390.0229812355, "avg_output_tokens_per_task": 46489.35800126502, "last_run_timestamp": "2026-10-10T08:26:02.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8436971598984772, "accuracy_std_dev": 0.01736385407066934, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.00971490228426396, "avg_duration_per_task_ms": 459655.7252538071, "avg_output_tokens_per_task": 30262.047588832487, "last_run_timestamp": "2026-10-10T08:38:35.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8436747142857144, "accuracy_std_dev": 0.038362541267358635, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1561614963924964, "avg_duration_per_task_ms": 1365496.6435786437, "avg_output_tokens_per_task": 51685.106782106785, "last_run_timestamp": "2026-10-10T16:24:14.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8435435936936937, "accuracy_std_dev": 0.023817522835089276, "run_count": 8, "total_questions": 1110, "avg_cost_per_task_usd": 0.015176712612612612, "avg_duration_per_task_ms": 82041.15225225226, "avg_output_tokens_per_task": 13011.012612612612, "last_run_timestamp": "2026-10-10T16:24:45.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8433861695238094, "accuracy_std_dev": 0.02133976042178954, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.04683724698412699, "avg_duration_per_task_ms": 634662.0095238095, "avg_output_tokens_per_task": 37604.758095238096, "last_run_timestamp": "2026-10-10T16:20:00.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8432812727272728, "accuracy_std_dev": 0.012668660387607238, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1344134775022957, "avg_duration_per_task_ms": 424945.45224977046, "avg_output_tokens_per_task": 37266.27272727273, "last_run_timestamp": "2026-10-10T08:22:18.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.843273301664355, "accuracy_std_dev": 0.026075377685423836, "run_count": 8, "total_questions": 1442, "avg_cost_per_task_usd": 0.010784043689320387, "avg_duration_per_task_ms": 367303.58252427186, "avg_output_tokens_per_task": 21354.485436893203, "last_run_timestamp": "2026-10-10T08:28:19.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8431938571428572, "accuracy_std_dev": 0.04476285754771171, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11224030519480518, "avg_duration_per_task_ms": 1310504.1515151516, "avg_output_tokens_per_task": 44998.9531024531, "last_run_timestamp": "2026-10-10T16:11:11.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8430194788376499, "accuracy_std_dev": 0.012894642985921526, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.1371234333543904, "avg_duration_per_task_ms": 198575.00252684776, "avg_output_tokens_per_task": 30938.104232469992, "last_run_timestamp": "2026-10-10T08:24:22.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8426664382222222, "accuracy_std_dev": 0.00961840561483182, "run_count": 7, "total_questions": 1125, "avg_cost_per_task_usd": 0.1371568737777778, "avg_duration_per_task_ms": 125791.95822222222, "avg_output_tokens_per_task": 30936.946666666667, "last_run_timestamp": "2026-10-10T08:24:22.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8419940950570343, "accuracy_std_dev": 0.005052124553096465, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.02034613561470215, "avg_duration_per_task_ms": 621368.3954372624, "avg_output_tokens_per_task": 23198.26235741445, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8414709276830954, "accuracy_std_dev": 0.01569670882911814, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.21642636665131276, "avg_duration_per_task_ms": 299666.9097190235, "avg_output_tokens_per_task": 49086.817134960846, "last_run_timestamp": "2026-10-10T08:18:24.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8413977096774194, "accuracy_std_dev": 0.007842951293996439, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.03688066532258065, "avg_duration_per_task_ms": 47685.283602150535, "avg_output_tokens_per_task": 14664.099462365592, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.8412499316666666, "accuracy_std_dev": 0.012659388006293123, "run_count": 7, "total_questions": 1200, "avg_cost_per_task_usd": 0.0068970816666666665, "avg_duration_per_task_ms": 636725.1641666667, "avg_output_tokens_per_task": 17029.4825, "last_run_timestamp": "2026-10-10T16:23:21.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8411896666666667, "accuracy_std_dev": 0.008729468916008341, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.036658867003367, "avg_duration_per_task_ms": 51092.77104377104, "avg_output_tokens_per_task": 14574.808080808081, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8408532168508287, "accuracy_std_dev": 0.05657006571975314, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.07583489180478822, "avg_duration_per_task_ms": 651639.9650092081, "avg_output_tokens_per_task": 45206.226979742176, "last_run_timestamp": "2026-10-10T08:22:51.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.8407408, "accuracy_std_dev": 0.1041355491182526, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11180628585858585, "avg_duration_per_task_ms": 1526581.008080808, "avg_output_tokens_per_task": 27815.140404040405, "last_run_timestamp": "2026-10-10T08:36:30.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.8407406999999999, "accuracy_std_dev": 0.01043760000239519, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07050250505050504, "avg_duration_per_task_ms": 363872.355050505, "avg_output_tokens_per_task": 22481.775757575757, "last_run_timestamp": "2026-10-10T08:46:09.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.84048825, "accuracy_std_dev": 0.004667830993887862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06197197222222223, "avg_duration_per_task_ms": 84839.70707070707, "avg_output_tokens_per_task": 13643.420454545454, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8402775531250001, "accuracy_std_dev": 0.007466846475587956, "run_count": 10, "total_questions": 1920, "avg_cost_per_task_usd": 0.039408469791666664, "avg_duration_per_task_ms": 51791.722395833334, "avg_output_tokens_per_task": 14757.431770833333, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8402425498181818, "accuracy_std_dev": 0.00976753775827742, "run_count": 8, "total_questions": 1375, "avg_cost_per_task_usd": 0.0796483170909091, "avg_duration_per_task_ms": 509723.63709090906, "avg_output_tokens_per_task": 19683.845818181817, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8401708461538463, "accuracy_std_dev": 0.01634594423702712, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.03290970448717949, "avg_duration_per_task_ms": 545901.551923077, "avg_output_tokens_per_task": 32901.243589743586, "last_run_timestamp": "2026-10-10T16:25:08.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8399772366412214, "accuracy_std_dev": 0.010642150299634022, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.02758528159457167, "avg_duration_per_task_ms": 524755.5767599661, "avg_output_tokens_per_task": 22800.932994062765, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8398466400000001, "accuracy_std_dev": 0.04372123622631255, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.16602242022988506, "avg_duration_per_task_ms": 162702.748045977, "avg_output_tokens_per_task": 16381.855172413792, "last_run_timestamp": "2026-10-10T08:14:40.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8394795097613883, "accuracy_std_dev": 0.06253985350713477, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.2298788691250904, "avg_duration_per_task_ms": 1437430.4526391902, "avg_output_tokens_per_task": 53441.45336225597, "last_run_timestamp": "2026-10-10T16:20:12.051Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8393612296564196, "accuracy_std_dev": 0.01918273370044586, "run_count": 6, "total_questions": 1106, "avg_cost_per_task_usd": 0.1468148047016275, "avg_duration_per_task_ms": 272284.95840867993, "avg_output_tokens_per_task": 34433.43580470163, "last_run_timestamp": "2026-10-10T08:21:50.326Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8391883298001213, "accuracy_std_dev": 0.103275715403127, "run_count": 17, "total_questions": 3302, "avg_cost_per_task_usd": 0.008241699273167776, "avg_duration_per_task_ms": 633419.09812235, "avg_output_tokens_per_task": 27504.60357359176, "last_run_timestamp": "2026-10-10T08:22:05.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8391142192408377, "accuracy_std_dev": 0.016722181240421387, "run_count": 8, "total_questions": 1528, "avg_cost_per_task_usd": 0.019121354712041884, "avg_duration_per_task_ms": 242204.53206806284, "avg_output_tokens_per_task": 38024.85013089005, "last_run_timestamp": "2026-10-10T08:22:13.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8388045000000002, "accuracy_std_dev": 0.02728541628782673, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.14381466666666667, "avg_duration_per_task_ms": 923603.7752525252, "avg_output_tokens_per_task": 54078.98484848485, "last_run_timestamp": "2026-10-10T16:23:01.146Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.838739, "accuracy_std_dev": null, "run_count": 1, "total_questions": 185, "avg_cost_per_task_usd": 0.0289266, "avg_duration_per_task_ms": 182176.34594594594, "avg_output_tokens_per_task": 24823.56216216216, "last_run_timestamp": "2026-07-18T08:31:24.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8386610687977762, "accuracy_std_dev": 0.022495213007090526, "run_count": 8, "total_questions": 1439, "avg_cost_per_task_usd": 0.07294077206393328, "avg_duration_per_task_ms": 384771.50799166085, "avg_output_tokens_per_task": 24034.94023627519, "last_run_timestamp": "2026-10-10T08:21:47.164Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8384554097714735, "accuracy_std_dev": 0.027250356683679502, "run_count": 7, "total_questions": 1269, "avg_cost_per_task_usd": 0.0903084074074074, "avg_duration_per_task_ms": 1604100.3364854215, "avg_output_tokens_per_task": 41888.97872340425, "last_run_timestamp": "2026-10-10T16:21:58.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.8383456936090224, "accuracy_std_dev": 0.05167297206793249, "run_count": 6, "total_questions": 1064, "avg_cost_per_task_usd": 0.17379162406015036, "avg_duration_per_task_ms": 1701245.0780075188, "avg_output_tokens_per_task": 51051.67105263158, "last_run_timestamp": "2026-10-10T16:20:06.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8379778156484459, "accuracy_std_dev": 0.014079857290552125, "run_count": 11, "total_questions": 1866, "avg_cost_per_task_usd": 0.09540400214362274, "avg_duration_per_task_ms": 623450.2524115755, "avg_output_tokens_per_task": 29739.112004287246, "last_run_timestamp": "2026-10-10T16:21:21.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8378226666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.15828378956228956, "avg_duration_per_task_ms": 446608.7861952862, "avg_output_tokens_per_task": 56298.93097643097, "last_run_timestamp": "2026-10-10T08:30:28.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8377989469160769, "accuracy_std_dev": 0.022618776770639035, "run_count": 10, "total_questions": 1978, "avg_cost_per_task_usd": 0.023259668857431744, "avg_duration_per_task_ms": 408040.1359959555, "avg_output_tokens_per_task": 43341.32406471183, "last_run_timestamp": "2026-10-10T08:39:09.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.837571, "accuracy_std_dev": null, "run_count": 1, "total_questions": 118, "avg_cost_per_task_usd": 0.01632071186440678, "avg_duration_per_task_ms": 269506.5254237288, "avg_output_tokens_per_task": 11788.906779661016, "last_run_timestamp": "2026-07-18T16:47:35.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8375589327830189, "accuracy_std_dev": 0.020056898198209345, "run_count": 9, "total_questions": 1696, "avg_cost_per_task_usd": 0.054286748820754724, "avg_duration_per_task_ms": 282159.9534198113, "avg_output_tokens_per_task": 22660.685141509435, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.836910625, "accuracy_std_dev": 0.02484845377954887, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.11375521527777778, "avg_duration_per_task_ms": 402363.48232323234, "avg_output_tokens_per_task": 35795.318181818184, "last_run_timestamp": "2026-10-10T08:22:05.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8368532727272727, "accuracy_std_dev": 0.07122412090410091, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.195708673553719, "avg_duration_per_task_ms": 540689.1230486685, "avg_output_tokens_per_task": 44503.561983471074, "last_run_timestamp": "2026-10-10T08:23:29.599Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8367697340206185, "accuracy_std_dev": 0.008932455147382513, "run_count": 5, "total_questions": 970, "avg_cost_per_task_usd": 0.027574227835051545, "avg_duration_per_task_ms": 610547.7175257732, "avg_output_tokens_per_task": 14641.665979381443, "last_run_timestamp": "2026-10-10T08:23:40.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.8367169999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.10272846700507614, "avg_duration_per_task_ms": 406781.3959390863, "avg_output_tokens_per_task": 31764.908629441623, "last_run_timestamp": "2026-07-18T08:30:34.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.8367000000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02250940404040404, "avg_duration_per_task_ms": 13191.318181818182, "avg_output_tokens_per_task": 14558.808080808081, "last_run_timestamp": "2026-10-10T08:14:48.050Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.8365800000000001, "accuracy_std_dev": 0.012840289382586837, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006623383116883117, "avg_duration_per_task_ms": 371056.68975468975, "avg_output_tokens_per_task": 26074.506493506495, "last_run_timestamp": "2026-10-10T16:19:34.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8364249544688027, "accuracy_std_dev": 0.025478198310237742, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.15890397301854978, "avg_duration_per_task_ms": 937364.9814502529, "avg_output_tokens_per_task": 52191.70320404722, "last_run_timestamp": "2026-10-10T16:25:51.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8363261111111111, "accuracy_std_dev": 0.010093985221621908, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.24586164983164982, "avg_duration_per_task_ms": 359650.97138047137, "avg_output_tokens_per_task": 40134.10662177329, "last_run_timestamp": "2026-10-10T08:34:27.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8362795, "accuracy_std_dev": 0.015277669703961197, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.07403895328282828, "avg_duration_per_task_ms": 689404.1414141414, "avg_output_tokens_per_task": 53986.75673400673, "last_run_timestamp": "2026-10-10T08:28:33.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8353882517845557, "accuracy_std_dev": 0.03954502959917974, "run_count": 8, "total_questions": 1541, "avg_cost_per_task_usd": 0.02963127903958469, "avg_duration_per_task_ms": 279716.5554834523, "avg_output_tokens_per_task": 9152.79299156392, "last_run_timestamp": "2026-10-10T08:50:00.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8353515086042066, "accuracy_std_dev": 0.02210739585947427, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.09059525812619502, "avg_duration_per_task_ms": 1011217.5761631612, "avg_output_tokens_per_task": 41548.43977055449, "last_run_timestamp": "2026-10-10T16:11:14.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8351813548597521, "accuracy_std_dev": 0.013967532009539117, "run_count": 8, "total_questions": 1533, "avg_cost_per_task_usd": 0.22283928962818, "avg_duration_per_task_ms": 428460.4801043705, "avg_output_tokens_per_task": 50510.284409654276, "last_run_timestamp": "2026-10-10T16:19:37.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8351770476190475, "accuracy_std_dev": 0.01163942668558729, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.2585508424723425, "avg_duration_per_task_ms": 272269.33116883115, "avg_output_tokens_per_task": 56376.68253968254, "last_run_timestamp": "2026-10-10T08:26:21.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8350167777777778, "accuracy_std_dev": 0.012294439174573386, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03943477890011223, "avg_duration_per_task_ms": 224321.7744107744, "avg_output_tokens_per_task": 14566.167227833894, "last_run_timestamp": "2026-10-10T08:14:46.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8350167272727272, "accuracy_std_dev": 0.011463201140177447, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.08175606887052343, "avg_duration_per_task_ms": 770659.5785123967, "avg_output_tokens_per_task": 48084.065197428834, "last_run_timestamp": "2026-10-10T16:30:58.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8350166666666667, "accuracy_std_dev": 0.019584493003961645, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.025601432659932657, "avg_duration_per_task_ms": 248785.20707070708, "avg_output_tokens_per_task": 21309.069865319867, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8350166666666667, "accuracy_std_dev": 0.005991526757754575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06194655555555556, "avg_duration_per_task_ms": 84396.87373737374, "avg_output_tokens_per_task": 13638.5, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8349160504017531, "accuracy_std_dev": 0.018287978469196532, "run_count": 7, "total_questions": 1369, "avg_cost_per_task_usd": 0.015230915997078159, "avg_duration_per_task_ms": 440198.80131482834, "avg_output_tokens_per_task": 39488.13586559532, "last_run_timestamp": "2026-10-10T08:48:34.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8348179217065166, "accuracy_std_dev": 0.020088025050917314, "run_count": 11, "total_questions": 2133, "avg_cost_per_task_usd": 0.21873191092358182, "avg_duration_per_task_ms": 294912.9784341303, "avg_output_tokens_per_task": 49538.53023909986, "last_run_timestamp": "2026-10-10T08:39:15.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8347633260247855, "accuracy_std_dev": 0.010241443659085482, "run_count": 6, "total_questions": 1049, "avg_cost_per_task_usd": 0.047060826501429925, "avg_duration_per_task_ms": 90755.15919923737, "avg_output_tokens_per_task": 15002.908484270734, "last_run_timestamp": "2026-10-10T08:22:14.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8346427777777778, "accuracy_std_dev": 0.050646967751239176, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1631596778900112, "avg_duration_per_task_ms": 161457.49102132436, "avg_output_tokens_per_task": 16154.749719416386, "last_run_timestamp": "2026-10-10T08:25:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8344556666666666, "accuracy_std_dev": 0.016627808360962375, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1539143468013468, "avg_duration_per_task_ms": 212926.7558922559, "avg_output_tokens_per_task": 34726.19528619529, "last_run_timestamp": "2026-10-10T08:23:16.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8342810781990522, "accuracy_std_dev": 0.009652908469324662, "run_count": 13, "total_questions": 2110, "avg_cost_per_task_usd": 0.018507007109004738, "avg_duration_per_task_ms": 188704.51184834124, "avg_output_tokens_per_task": 36823.764454976306, "last_run_timestamp": "2026-10-10T08:36:35.871Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "WandB", "accuracy": 0.834197, "accuracy_std_dev": null, "run_count": 1, "total_questions": 193, "avg_cost_per_task_usd": 0.02599202590673575, "avg_duration_per_task_ms": 128468.99481865285, "avg_output_tokens_per_task": 20660.492227979274, "last_run_timestamp": "2026-07-18T16:00:08.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8341792639593909, "accuracy_std_dev": 0.020282949394476377, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.07842123350253806, "avg_duration_per_task_ms": 728101.9196277496, "avg_output_tokens_per_task": 42337.8130287648, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.833648875, "accuracy_std_dev": 0.013388255986474674, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13926744002525251, "avg_duration_per_task_ms": 187514.4393939394, "avg_output_tokens_per_task": 31415.290404040403, "last_run_timestamp": "2026-10-10T08:23:04.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8334175866261399, "accuracy_std_dev": 0.031437998183885686, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.01607202178318136, "avg_duration_per_task_ms": 132425.65501519758, "avg_output_tokens_per_task": 17865.155015197568, "last_run_timestamp": "2026-10-10T16:25:38.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.8334175326251898, "accuracy_std_dev": 0.0340950993933146, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.10936984522003032, "avg_duration_per_task_ms": 1005533.760748609, "avg_output_tokens_per_task": 34172.51643904906, "last_run_timestamp": "2026-10-10T16:20:24.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.8333335829015545, "accuracy_std_dev": 0.019006499999999982, "run_count": 2, "total_questions": 386, "avg_cost_per_task_usd": 0.023828624352331605, "avg_duration_per_task_ms": 290054.69430051814, "avg_output_tokens_per_task": 19802.816062176167, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8333334266055047, "accuracy_std_dev": 0.015888679779606903, "run_count": 6, "total_questions": 1090, "avg_cost_per_task_usd": 0.02116503211009174, "avg_duration_per_task_ms": 295187.03211009176, "avg_output_tokens_per_task": 42108.07706422018, "last_run_timestamp": "2026-10-10T08:23:44.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8332223213453214, "accuracy_std_dev": 0.019068183116368872, "run_count": 16, "total_questions": 3003, "avg_cost_per_task_usd": 0.19684398967698968, "avg_duration_per_task_ms": 375126.0233100233, "avg_output_tokens_per_task": 39605.68764568765, "last_run_timestamp": "2026-10-10T08:22:27.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8331929166666667, "accuracy_std_dev": 0.03339137345547584, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.054762767255892256, "avg_duration_per_task_ms": 267512.9158249158, "avg_output_tokens_per_task": 23368.800084175084, "last_run_timestamp": "2026-10-10T08:22:25.200Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8331589885057472, "accuracy_std_dev": 0.013081913080821427, "run_count": 12, "total_questions": 1914, "avg_cost_per_task_usd": 0.11308331609195402, "avg_duration_per_task_ms": 223576.03500522466, "avg_output_tokens_per_task": 27212.787356321838, "last_run_timestamp": "2026-10-10T08:14:48.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.8325081287128712, "accuracy_std_dev": 0.06854471399887452, "run_count": 4, "total_questions": 606, "avg_cost_per_task_usd": 0.13443122607260727, "avg_duration_per_task_ms": 1513362.3762376239, "avg_output_tokens_per_task": 46533.88613861386, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8324914999999999, "accuracy_std_dev": 0.01902788623310536, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02279124747474747, "avg_duration_per_task_ms": 532516.4444444445, "avg_output_tokens_per_task": 41107.4779040404, "last_run_timestamp": "2026-10-03T10:07:54.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8320707500000001, "accuracy_std_dev": 0.05130359405721493, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09154002398989898, "avg_duration_per_task_ms": 446836.672979798, "avg_output_tokens_per_task": 20695.150883838385, "last_run_timestamp": "2026-10-10T16:11:11.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8314910184229919, "accuracy_std_dev": 0.014032999291029713, "run_count": 7, "total_questions": 1357, "avg_cost_per_task_usd": 0.15096693072955047, "avg_duration_per_task_ms": 266519.0810611643, "avg_output_tokens_per_task": 34117.86366985999, "last_run_timestamp": "2026-10-10T08:23:25.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8310885000000001, "accuracy_std_dev": 0.012955632388399007, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06418621717171719, "avg_duration_per_task_ms": 117116.24831649831, "avg_output_tokens_per_task": 13483.090067340067, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8309619512195122, "accuracy_std_dev": 0.016029553150759313, "run_count": 13, "total_questions": 1968, "avg_cost_per_task_usd": 0.036442337906504065, "avg_duration_per_task_ms": 251079.88109756098, "avg_output_tokens_per_task": 29050.197662601626, "last_run_timestamp": "2026-10-10T16:21:24.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8307314545454545, "accuracy_std_dev": 0.026853345034104435, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.13328792148760332, "avg_duration_per_task_ms": 548052.8255280073, "avg_output_tokens_per_task": 41331.14049586777, "last_run_timestamp": "2026-10-10T08:14:46.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.830415961308153, "accuracy_std_dev": 0.034463455713538205, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.06569626301243668, "avg_duration_per_task_ms": 1046585.6020267158, "avg_output_tokens_per_task": 33076.09580838323, "last_run_timestamp": "2026-10-10T16:21:31.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.8303493580562661, "accuracy_std_dev": 0.021497934689561845, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.0706463444160273, "avg_duration_per_task_ms": 871151.2676896845, "avg_output_tokens_per_task": 40306.14833759591, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8301192727272728, "accuracy_std_dev": 0.022568976244920873, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.23170999265381084, "avg_duration_per_task_ms": 339625.2497704316, "avg_output_tokens_per_task": 52536.71900826446, "last_run_timestamp": "2026-10-10T08:22:18.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.8296903524590165, "accuracy_std_dev": 0.03798493151237738, "run_count": 3, "total_questions": 366, "avg_cost_per_task_usd": 0.08753724316939891, "avg_duration_per_task_ms": 313974.91803278687, "avg_output_tokens_per_task": 36231.311475409835, "last_run_timestamp": "2026-10-10T16:20:11.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8295325781071835, "accuracy_std_dev": 0.006460861466467855, "run_count": 9, "total_questions": 1754, "avg_cost_per_task_usd": 0.035354884834663626, "avg_duration_per_task_ms": 201763.31527936147, "avg_output_tokens_per_task": 14117.418472063853, "last_run_timestamp": "2026-10-10T08:15:01.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8292925999999999, "accuracy_std_dev": 0.004466830267650664, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.057570920202020204, "avg_duration_per_task_ms": 214582.1888888889, "avg_output_tokens_per_task": 15222.584848484848, "last_run_timestamp": "2026-10-10T08:19:01.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8290087877529839, "accuracy_std_dev": 0.02323794562004137, "run_count": 10, "total_questions": 1927, "avg_cost_per_task_usd": 0.02390980435910742, "avg_duration_per_task_ms": 846858.6066424494, "avg_output_tokens_per_task": 46042.94706798132, "last_run_timestamp": "2026-10-10T08:17:16.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.8288439999999999, "accuracy_std_dev": 0.005203901805376433, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02148271043771044, "avg_duration_per_task_ms": 661167.3181818182, "avg_output_tokens_per_task": 55670.02693602694, "last_run_timestamp": "2026-10-10T08:22:14.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.828283, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009031212121212122, "avg_duration_per_task_ms": 399484.5707070707, "avg_output_tokens_per_task": 21544.277777777777, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8280361855845629, "accuracy_std_dev": 0.008498377450075449, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.007634015323496027, "avg_duration_per_task_ms": 329436.5028376845, "avg_output_tokens_per_task": 19170.439841089672, "last_run_timestamp": "2026-10-10T16:21:04.886Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8276705454545453, "accuracy_std_dev": 0.013998517625180167, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.06217915610651974, "avg_duration_per_task_ms": 529389.9205693296, "avg_output_tokens_per_task": 48974.31083562902, "last_run_timestamp": "2026-10-10T16:25:44.293Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8275666632911391, "accuracy_std_dev": 0.031862730949862604, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.025537988185654007, "avg_duration_per_task_ms": 492003.4506329114, "avg_output_tokens_per_task": 23813.98734177215, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.827288, "accuracy_std_dev": 0.05899557909168817, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.05165088016528926, "avg_duration_per_task_ms": 574742.4444444445, "avg_output_tokens_per_task": 24729.172635445364, "last_run_timestamp": "2026-10-10T08:28:19.656Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8272306250000001, "accuracy_std_dev": 0.010511419182697223, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04333868371212121, "avg_duration_per_task_ms": 227486.86805555556, "avg_output_tokens_per_task": 14329.91856060606, "last_run_timestamp": "2026-10-10T08:22:04.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.827217112385321, "accuracy_std_dev": 0.014186141135664356, "run_count": 9, "total_questions": 1744, "avg_cost_per_task_usd": 0.036017568233944953, "avg_duration_per_task_ms": 436504.3365825688, "avg_output_tokens_per_task": 47925.99369266055, "last_run_timestamp": "2026-10-10T08:14:40.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8265992857142858, "accuracy_std_dev": 0.05408198806498543, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09954717027417027, "avg_duration_per_task_ms": 788800.0598845598, "avg_output_tokens_per_task": 39778.419913419915, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8265991666666667, "accuracy_std_dev": 0.013432985736817995, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024518459595959594, "avg_duration_per_task_ms": 477551.41245791246, "avg_output_tokens_per_task": 22795.994107744107, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.8265989999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02378419696969697, "avg_duration_per_task_ms": 421976.08585858584, "avg_output_tokens_per_task": 20387.959595959597, "last_run_timestamp": "2026-07-18T16:18:27.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8263887499999999, "accuracy_std_dev": 0.018961582467913917, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12611158901515152, "avg_duration_per_task_ms": 326364.59911616164, "avg_output_tokens_per_task": 52045.35858585859, "last_run_timestamp": "2026-10-10T08:22:18.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8263887499999999, "accuracy_std_dev": 0.0068091963686987495, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13881603345959595, "avg_duration_per_task_ms": 161822.7279040404, "avg_output_tokens_per_task": 34223.880681818184, "last_run_timestamp": "2026-10-10T08:21:09.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8262248861473387, "accuracy_std_dev": 0.021314932165659498, "run_count": 15, "total_questions": 2837, "avg_cost_per_task_usd": 0.04278339231582658, "avg_duration_per_task_ms": 372895.77722946776, "avg_output_tokens_per_task": 46085.86993302785, "last_run_timestamp": "2026-10-10T16:22:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8258873707224335, "accuracy_std_dev": 0.009676826643527784, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.025350776615969583, "avg_duration_per_task_ms": 120122.08840304182, "avg_output_tokens_per_task": 20151.38783269962, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8258778571428571, "accuracy_std_dev": 0.01206814403328714, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03852732972582973, "avg_duration_per_task_ms": 268250.60461760464, "avg_output_tokens_per_task": 15536.08658008658, "last_run_timestamp": "2026-10-10T08:21:53.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8258626249999999, "accuracy_std_dev": 0.012410001157710468, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09867501325757576, "avg_duration_per_task_ms": 367182.26704545453, "avg_output_tokens_per_task": 45431.65340909091, "last_run_timestamp": "2026-10-10T16:11:14.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.8255893999999999, "accuracy_std_dev": 0.010301039221360171, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.14328596767676768, "avg_duration_per_task_ms": 412091.31313131313, "avg_output_tokens_per_task": 28667.273737373736, "last_run_timestamp": "2026-10-10T08:24:21.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8254515454545454, "accuracy_std_dev": 0.024386361607048986, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.037847392102846655, "avg_duration_per_task_ms": 348957.9407713499, "avg_output_tokens_per_task": 39061.49908172635, "last_run_timestamp": "2026-10-10T16:22:04.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8253399469387755, "accuracy_std_dev": 0.06202361079266511, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.048117852040816325, "avg_duration_per_task_ms": 1077180.0714285714, "avg_output_tokens_per_task": 26155.976530612246, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8250688181818181, "accuracy_std_dev": 0.022841304837819883, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.010740173094582185, "avg_duration_per_task_ms": 682886.4674012855, "avg_output_tokens_per_task": 38122.29706152433, "last_run_timestamp": "2026-10-10T08:14:49.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.824916, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022821656565656566, "avg_duration_per_task_ms": 516406.73232323234, "avg_output_tokens_per_task": 45403.40909090909, "last_run_timestamp": "2026-10-04T03:04:38.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8248312595648912, "accuracy_std_dev": 0.01663705107798342, "run_count": 7, "total_questions": 1333, "avg_cost_per_task_usd": 0.02528466016504126, "avg_duration_per_task_ms": 296846.2640660165, "avg_output_tokens_per_task": 50351.12678169542, "last_run_timestamp": "2026-10-10T08:35:20.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.8243470488301118, "accuracy_std_dev": 0.01897362841419636, "run_count": 5, "total_questions": 983, "avg_cost_per_task_usd": 0.09452765412004069, "avg_duration_per_task_ms": 201477.20142421158, "avg_output_tokens_per_task": 27045.996948118005, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8242846250000001, "accuracy_std_dev": 0.007803239774886767, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04825054608585859, "avg_duration_per_task_ms": 444851.5814393939, "avg_output_tokens_per_task": 26726.752525252527, "last_run_timestamp": "2026-10-10T16:10:48.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8233164, "accuracy_std_dev": 0.059578276225819106, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.027962910606060605, "avg_duration_per_task_ms": 676433.6525252525, "avg_output_tokens_per_task": 29240.15101010101, "last_run_timestamp": "2026-10-10T08:21:58.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025912994949494946, "avg_duration_per_task_ms": 162312.898989899, "avg_output_tokens_per_task": 9036.257575757576, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8231511800643088, "accuracy_std_dev": 0.016091623665357716, "run_count": 8, "total_questions": 1555, "avg_cost_per_task_usd": 0.14175151639871383, "avg_duration_per_task_ms": 272966.68681672023, "avg_output_tokens_per_task": 31995.940836012862, "last_run_timestamp": "2026-10-10T08:25:06.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.823129, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.032144397959183675, "avg_duration_per_task_ms": 345053.9591836735, "avg_output_tokens_per_task": 27884.80612244898, "last_run_timestamp": "2026-07-18T08:29:12.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8229467130434782, "accuracy_std_dev": 0.020047430385493913, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.006798507246376812, "avg_duration_per_task_ms": 630419.7739130434, "avg_output_tokens_per_task": 18294.628260869566, "last_run_timestamp": "2026-10-10T16:19:44.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8224291477721376, "accuracy_std_dev": 0.010721893131391866, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.025065345741680765, "avg_duration_per_task_ms": 452443.4557247603, "avg_output_tokens_per_task": 25021.49464184997, "last_run_timestamp": "2026-10-10T16:11:11.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007053464646464647, "avg_duration_per_task_ms": 622803.648989899, "avg_output_tokens_per_task": 16887.560606060608, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8223905, "accuracy_std_dev": 0.021663157431839608, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10283124074074075, "avg_duration_per_task_ms": 597427.9688552188, "avg_output_tokens_per_task": 38154.76262626263, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8222970000000001, "accuracy_std_dev": 0.02180780391918859, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03627528900112234, "avg_duration_per_task_ms": 268542.3265993266, "avg_output_tokens_per_task": 14420.646464646465, "last_run_timestamp": "2026-10-10T08:43:05.187Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8217018181818182, "accuracy_std_dev": 0.0209343466485894, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.012272595959595959, "avg_duration_per_task_ms": 695475.8053259872, "avg_output_tokens_per_task": 36304.134527089074, "last_run_timestamp": "2026-10-10T08:14:48.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8215404398976982, "accuracy_std_dev": 0.032891408388902345, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.02587831713554987, "avg_duration_per_task_ms": 460054.41943734017, "avg_output_tokens_per_task": 21496.38874680307, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8212957816756088, "accuracy_std_dev": 0.017294239247104652, "run_count": 13, "total_questions": 2423, "avg_cost_per_task_usd": 0.031653728023111845, "avg_duration_per_task_ms": 257591.64630623194, "avg_output_tokens_per_task": 14017.31324803962, "last_run_timestamp": "2026-10-10T08:14:46.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.8212274003378379, "accuracy_std_dev": 0.0061029639361178676, "run_count": 6, "total_questions": 1184, "avg_cost_per_task_usd": 0.039089869087837834, "avg_duration_per_task_ms": 209017.77111486485, "avg_output_tokens_per_task": 14834.131756756757, "last_run_timestamp": "2026-10-10T08:21:53.663Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.821210220113852, "accuracy_std_dev": 0.012972061902411654, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.19423610879190387, "avg_duration_per_task_ms": 987635.4212523719, "avg_output_tokens_per_task": 51696.76660341556, "last_run_timestamp": "2026-10-10T16:21:24.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8211085620915033, "accuracy_std_dev": 0.010055056516550273, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.022989802469135803, "avg_duration_per_task_ms": 339486.9368191721, "avg_output_tokens_per_task": 20491.059549745823, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8208700266159695, "accuracy_std_dev": 0.003273364894779542, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.007889804816223067, "avg_duration_per_task_ms": 461257.1007604563, "avg_output_tokens_per_task": 20299.61216730038, "last_run_timestamp": "2026-10-10T16:10:51.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.8208440232558138, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 387, "avg_cost_per_task_usd": 0.007978992248062014, "avg_duration_per_task_ms": 646108.8320413437, "avg_output_tokens_per_task": 19074.485788113696, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.8205865714285715, "accuracy_std_dev": 0.030540716690332988, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09537167965367965, "avg_duration_per_task_ms": 1474899.9105339106, "avg_output_tokens_per_task": 44220.89033189033, "last_run_timestamp": "2026-10-10T16:26:06.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8205389659969088, "accuracy_std_dev": 0.05440642468688786, "run_count": 10, "total_questions": 1941, "avg_cost_per_task_usd": 0.08737539618753219, "avg_duration_per_task_ms": 555621.9278722308, "avg_output_tokens_per_task": 29961.489438433797, "last_run_timestamp": "2026-10-10T08:48:03.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8202530101265821, "accuracy_std_dev": 0.008020329102765456, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.018872242194092826, "avg_duration_per_task_ms": 516686.08607594937, "avg_output_tokens_per_task": 18743.98734177215, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.8198651666666666, "accuracy_std_dev": 0.10272117659862329, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.17633816414141412, "avg_duration_per_task_ms": 1166115.4705387205, "avg_output_tokens_per_task": 47461.85521885522, "last_run_timestamp": "2026-10-10T16:33:35.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.819865, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008803729797979798, "avg_duration_per_task_ms": 1022564.7878787878, "avg_output_tokens_per_task": 21115.48232323232, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8195591818181818, "accuracy_std_dev": 0.014154784085686241, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.31459056198347113, "avg_duration_per_task_ms": 289293.8429752066, "avg_output_tokens_per_task": 20783.41092745638, "last_run_timestamp": "2026-10-10T08:22:14.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.819444375, "accuracy_std_dev": 0.015469708868765922, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03466034659090909, "avg_duration_per_task_ms": 238239.72095959596, "avg_output_tokens_per_task": 13747.289772727272, "last_run_timestamp": "2026-10-10T08:21:42.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8192329035532995, "accuracy_std_dev": 0.01830389154129678, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.022038197123519458, "avg_duration_per_task_ms": 499412.81133671745, "avg_output_tokens_per_task": 22395.89847715736, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8191919000000001, "accuracy_std_dev": 0.019432571659201477, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04233793585858586, "avg_duration_per_task_ms": 245621.01414141414, "avg_output_tokens_per_task": 14127.980303030303, "last_run_timestamp": "2026-10-10T08:23:34.137Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8190038027586207, "accuracy_std_dev": 0.04321943578037032, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.0850050151724138, "avg_duration_per_task_ms": 798913.6754022988, "avg_output_tokens_per_task": 31455.114942528737, "last_run_timestamp": "2026-10-10T16:23:10.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8187577756653992, "accuracy_std_dev": 0.007032873860263311, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.008747721166032953, "avg_duration_per_task_ms": 704234.403041825, "avg_output_tokens_per_task": 18267.351077313055, "last_run_timestamp": "2026-10-10T16:19:34.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8184820903010034, "accuracy_std_dev": 0.06228079588716609, "run_count": 22, "total_questions": 4186, "avg_cost_per_task_usd": 0.007251606784519827, "avg_duration_per_task_ms": 497071.1039178213, "avg_output_tokens_per_task": 17273.298853320593, "last_run_timestamp": "2026-10-10T16:27:03.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8180962405063291, "accuracy_std_dev": 0.020585850087863757, "run_count": 5, "total_questions": 711, "avg_cost_per_task_usd": 0.10679723066104078, "avg_duration_per_task_ms": 387578.5597749648, "avg_output_tokens_per_task": 44272.65400843882, "last_run_timestamp": "2026-10-10T16:20:25.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.817901, "accuracy_std_dev": null, "run_count": 1, "total_questions": 54, "avg_cost_per_task_usd": 0.008988981481481481, "avg_duration_per_task_ms": 114091.12962962964, "avg_output_tokens_per_task": 16251.87037037037, "last_run_timestamp": "2026-07-18T09:01:08.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8168565855513307, "accuracy_std_dev": 0.03334781637130645, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.08050372496831433, "avg_duration_per_task_ms": 908697.1989860584, "avg_output_tokens_per_task": 40687.09125475285, "last_run_timestamp": "2026-10-10T16:10:34.759Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8167003199999999, "accuracy_std_dev": 0.011578936088328655, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.3337196343434343, "avg_duration_per_task_ms": 298714.2325252525, "avg_output_tokens_per_task": 21127.507474747475, "last_run_timestamp": "2026-10-10T08:23:34.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8162174999999999, "accuracy_std_dev": 0.009642570572725913, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.019066546296296293, "avg_duration_per_task_ms": 525999.5555555555, "avg_output_tokens_per_task": 18868.49074074074, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.815937, "accuracy_std_dev": 0.021042110493009022, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04137441245791246, "avg_duration_per_task_ms": 82874.20875420875, "avg_output_tokens_per_task": 27465.196969696968, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "WandB", "accuracy": 0.815838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 181, "avg_cost_per_task_usd": 0.03376955248618784, "avg_duration_per_task_ms": 178211.5359116022, "avg_output_tokens_per_task": 26893.3591160221, "last_run_timestamp": "2026-07-18T08:04:57.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8157000210325048, "accuracy_std_dev": 0.029811887613993424, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.2467402931803696, "avg_duration_per_task_ms": 963446.5092415551, "avg_output_tokens_per_task": 56213.97705544933, "last_run_timestamp": "2026-10-10T16:11:11.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8156999235181644, "accuracy_std_dev": 0.04951986238986445, "run_count": 9, "total_questions": 1569, "avg_cost_per_task_usd": 0.04647088910133843, "avg_duration_per_task_ms": 572516.0726577438, "avg_output_tokens_per_task": 26962.156787762906, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8155362857142857, "accuracy_std_dev": 0.02021913729058464, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0062872914862914875, "avg_duration_per_task_ms": 495775.87012987013, "avg_output_tokens_per_task": 18627.747474747473, "last_run_timestamp": "2026-10-10T16:21:33.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8155034545454546, "accuracy_std_dev": 0.00991632568374574, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.3643461124885216, "avg_duration_per_task_ms": 358943.8365472911, "avg_output_tokens_per_task": 22948.309228650138, "last_run_timestamp": "2026-10-10T08:22:14.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8152958571428572, "accuracy_std_dev": 0.016658861214968797, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006618044011544012, "avg_duration_per_task_ms": 520273.7655122655, "avg_output_tokens_per_task": 17626.31096681097, "last_run_timestamp": "2026-10-10T16:21:38.183Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8152955714285716, "accuracy_std_dev": 0.00996837593674748, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.033789289321789324, "avg_duration_per_task_ms": 681304.6854256854, "avg_output_tokens_per_task": 15943.984848484848, "last_run_timestamp": "2026-10-10T08:14:49.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8149562466539196, "accuracy_std_dev": 0.025669785816390737, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.20883088782664117, "avg_duration_per_task_ms": 930609.0956022944, "avg_output_tokens_per_task": 55931.00191204589, "last_run_timestamp": "2026-10-10T16:20:08.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8148146250000001, "accuracy_std_dev": 0.021706775042239102, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0411005606060606, "avg_duration_per_task_ms": 178924.70391414143, "avg_output_tokens_per_task": 22731.602272727272, "last_run_timestamp": "2026-10-10T16:10:48.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8142806714727086, "accuracy_std_dev": 0.02584273780697392, "run_count": 5, "total_questions": 971, "avg_cost_per_task_usd": 0.06495255406797117, "avg_duration_per_task_ms": 232305.07929969105, "avg_output_tokens_per_task": 18958.6982492276, "last_run_timestamp": "2026-10-10T08:46:13.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8141833749999999, "accuracy_std_dev": 0.00914133656717522, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02917629608585859, "avg_duration_per_task_ms": 198512.0467171717, "avg_output_tokens_per_task": 23747.22474747475, "last_run_timestamp": "2026-10-10T16:21:37.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.814024, "accuracy_std_dev": null, "run_count": 1, "total_questions": 164, "avg_cost_per_task_usd": 0.010100225609756097, "avg_duration_per_task_ms": 515065.23780487804, "avg_output_tokens_per_task": 33245.84146341463, "last_run_timestamp": "2026-09-28T09:46:11.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8139730000000001, "accuracy_std_dev": 0.007059363734312956, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006141271885521885, "avg_duration_per_task_ms": 521938.39983164985, "avg_output_tokens_per_task": 18689.60521885522, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8135448247619047, "accuracy_std_dev": 0.049998493601682646, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.22882532634920633, "avg_duration_per_task_ms": 1050315.3606349207, "avg_output_tokens_per_task": 52425.60063492064, "last_run_timestamp": "2026-10-10T16:22:35.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8131313333333333, "accuracy_std_dev": 0.02212166805545088, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2550713585858586, "avg_duration_per_task_ms": 227479.37878787878, "avg_output_tokens_per_task": 25491.23232323232, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.8131312857142856, "accuracy_std_dev": 0.08184342879584912, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011368694805194807, "avg_duration_per_task_ms": 741530.3095238095, "avg_output_tokens_per_task": 25627.320346320346, "last_run_timestamp": "2026-10-10T16:19:34.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.811973875, "accuracy_std_dev": 0.028210901188359352, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01455338888888889, "avg_duration_per_task_ms": 917219.5435606061, "avg_output_tokens_per_task": 43021.98421717172, "last_run_timestamp": "2026-10-10T08:36:08.861Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.811448, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004932363636363637, "avg_duration_per_task_ms": 1604879.5959595959, "avg_output_tokens_per_task": 24145.00505050505, "last_run_timestamp": "2026-07-18T08:42:46.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8114476666666668, "accuracy_std_dev": 0.026594828950171668, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11157255948372617, "avg_duration_per_task_ms": 774642.1705948373, "avg_output_tokens_per_task": 58026.93097643097, "last_run_timestamp": "2026-10-10T16:20:12.057Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8110362107795959, "accuracy_std_dev": 0.03156884697855712, "run_count": 6, "total_questions": 1039, "avg_cost_per_task_usd": 0.07300517420596726, "avg_duration_per_task_ms": 1437301.4898941289, "avg_output_tokens_per_task": 41806.461982675646, "last_run_timestamp": "2026-10-10T16:10:51.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8103253333333333, "accuracy_std_dev": 0.011527714209195522, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04107354882154882, "avg_duration_per_task_ms": 75169.15151515152, "avg_output_tokens_per_task": 27258.996632996634, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.810185, "accuracy_std_dev": 0.02559748825568634, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008685453914141415, "avg_duration_per_task_ms": 555085.3888888889, "avg_output_tokens_per_task": 18946.522095959597, "last_run_timestamp": "2026-10-10T16:10:48.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8091166153846153, "accuracy_std_dev": 0.03183613126622014, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.19830280854700855, "avg_duration_per_task_ms": 1060541.5008547008, "avg_output_tokens_per_task": 55592.3264957265, "last_run_timestamp": "2026-10-03T16:58:49.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8089225, "accuracy_std_dev": 0.015608773342258514, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.34001660497835495, "avg_duration_per_task_ms": 354284.26948051946, "avg_output_tokens_per_task": 22498.079725829724, "last_run_timestamp": "2026-10-10T08:22:14.582Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8087552731239093, "accuracy_std_dev": 0.0546199990705989, "run_count": 6, "total_questions": 1146, "avg_cost_per_task_usd": 0.024335609075043632, "avg_duration_per_task_ms": 1150538.0907504363, "avg_output_tokens_per_task": 49299.98254799302, "last_run_timestamp": "2026-10-03T10:11:21.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.808081, "accuracy_std_dev": 0.018027342695657245, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.044340436026936034, "avg_duration_per_task_ms": 217888.1734006734, "avg_output_tokens_per_task": 22075.030303030304, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.807927818181818, "accuracy_std_dev": 0.021837768516611848, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.3410933209366391, "avg_duration_per_task_ms": 355919.63177226816, "avg_output_tokens_per_task": 22581.024334251608, "last_run_timestamp": "2026-10-10T08:25:00.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.8075253162083936, "accuracy_std_dev": 0.05075034301643404, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.19541867366136037, "avg_duration_per_task_ms": 620138.4428364689, "avg_output_tokens_per_task": 46512.787264833576, "last_run_timestamp": "2026-10-10T16:19:34.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.807420008605852, "accuracy_std_dev": 0.03225256635592289, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.017075846815834766, "avg_duration_per_task_ms": 580474.0820424556, "avg_output_tokens_per_task": 18206.01204819277, "last_run_timestamp": "2026-10-10T16:11:14.298Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.8066155190839694, "accuracy_std_dev": 0.009958499999999981, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07586704071246819, "avg_duration_per_task_ms": 385930.98473282444, "avg_output_tokens_per_task": 32444.25445292621, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8063972857142858, "accuracy_std_dev": 0.04410277557094591, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04038211976911977, "avg_duration_per_task_ms": 957664.380952381, "avg_output_tokens_per_task": 25231.96176046176, "last_run_timestamp": "2026-09-26T09:34:13.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.806210222222222, "accuracy_std_dev": 0.03697273211969953, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.25474633557800225, "avg_duration_per_task_ms": 231596.29349046017, "avg_output_tokens_per_task": 25466.63243546577, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8061798193604147, "accuracy_std_dev": 0.10627998128250551, "run_count": 12, "total_questions": 2314, "avg_cost_per_task_usd": 0.04518147709593777, "avg_duration_per_task_ms": 176487.23595505618, "avg_output_tokens_per_task": 21622.97363872083, "last_run_timestamp": "2026-10-10T08:21:29.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.8058417989690723, "accuracy_std_dev": 0.019860476718151096, "run_count": 6, "total_questions": 1164, "avg_cost_per_task_usd": 0.1695892783505155, "avg_duration_per_task_ms": 882585.4982817869, "avg_output_tokens_per_task": 53988.948453608245, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.8053260000000001, "accuracy_std_dev": 0.037748749347133695, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.11406567493112948, "avg_duration_per_task_ms": 444708.36363636365, "avg_output_tokens_per_task": 31880.327364554636, "last_run_timestamp": "2026-10-10T08:22:15.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8051542427745665, "accuracy_std_dev": 0.11605686870967129, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.12536348916184972, "avg_duration_per_task_ms": 697081.9812138728, "avg_output_tokens_per_task": 39514.423410404626, "last_run_timestamp": "2026-10-10T16:19:37.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.805068, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.07262640935672515, "avg_duration_per_task_ms": 352658, "avg_output_tokens_per_task": 15572.333333333334, "last_run_timestamp": "2026-07-18T08:30:38.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8039654444444444, "accuracy_std_dev": 0.022307357058010714, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.3361001139169472, "avg_duration_per_task_ms": 353464.27497194166, "avg_output_tokens_per_task": 22284.79012345679, "last_run_timestamp": "2026-10-10T08:23:05.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.8032717017208413, "accuracy_std_dev": 0.028736543137614867, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.258291845124283, "avg_duration_per_task_ms": 931434.8087954111, "avg_output_tokens_per_task": 65094.10834926705, "last_run_timestamp": "2026-10-10T16:19:34.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.8024693333333334, "accuracy_std_dev": 0.005722799567422317, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.054554207070707066, "avg_duration_per_task_ms": 52570.52356902357, "avg_output_tokens_per_task": 15236.12962962963, "last_run_timestamp": "2026-10-10T08:22:47.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8024180909090908, "accuracy_std_dev": 0.10219243254179791, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09314315335169882, "avg_duration_per_task_ms": 521856.9338842975, "avg_output_tokens_per_task": 34359.95913682277, "last_run_timestamp": "2026-10-10T16:22:50.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8020951111111111, "accuracy_std_dev": 0.027492606765554845, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04329651571268237, "avg_duration_per_task_ms": 84651.98765432098, "avg_output_tokens_per_task": 26957.542087542086, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.8016821288180611, "accuracy_std_dev": 0.026730709555864766, "run_count": 5, "total_questions": 753, "avg_cost_per_task_usd": 0.050311730411686593, "avg_duration_per_task_ms": 763057.3041168659, "avg_output_tokens_per_task": 21804.07835325365, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.8011870934718102, "accuracy_std_dev": 0.030695975976321205, "run_count": 6, "total_questions": 674, "avg_cost_per_task_usd": 0.030323706231454008, "avg_duration_per_task_ms": 103892.11572700297, "avg_output_tokens_per_task": 31484.501483679524, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.8009257499999999, "accuracy_std_dev": 0.04236236185209107, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.026970955808080805, "avg_duration_per_task_ms": 357603.6609848485, "avg_output_tokens_per_task": 25554.109217171717, "last_run_timestamp": "2026-10-10T16:10:34.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.8008437316455697, "accuracy_std_dev": 0.022930653266364256, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.028163265822784812, "avg_duration_per_task_ms": 479213.82784810127, "avg_output_tokens_per_task": 23504.80253164557, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.8007853333333332, "accuracy_std_dev": 0.008398773614178595, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0439676531986532, "avg_duration_per_task_ms": 201662.1632996633, "avg_output_tokens_per_task": 21888.12962962963, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.8006493194805195, "accuracy_std_dev": 0.04527692613517397, "run_count": 4, "total_questions": 770, "avg_cost_per_task_usd": 0.00850762987012987, "avg_duration_per_task_ms": 622509.1441558441, "avg_output_tokens_per_task": 19295.080519480518, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7996631666666666, "accuracy_std_dev": 0.024569953550469384, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.045980551346801346, "avg_duration_per_task_ms": 194150.27525252526, "avg_output_tokens_per_task": 21808.128787878788, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.799032, "accuracy_std_dev": 0.02239639471544471, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.028761780303030306, "avg_duration_per_task_ms": 300265.7859848485, "avg_output_tokens_per_task": 19022.03345959596, "last_run_timestamp": "2026-10-10T16:11:11.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.7988215, "accuracy_std_dev": 0.04292950000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02789190404040404, "avg_duration_per_task_ms": 202775.87121212122, "avg_output_tokens_per_task": 12888.320707070707, "last_run_timestamp": "2026-10-10T08:25:24.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7979796666666668, "accuracy_std_dev": 0.009816464683830371, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01347027946127946, "avg_duration_per_task_ms": 287878.5387205387, "avg_output_tokens_per_task": 14030.314814814816, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.797481, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.07670138953488372, "avg_duration_per_task_ms": 240811.48837209304, "avg_output_tokens_per_task": 21222.406976744187, "last_run_timestamp": "2026-07-18T08:37:00.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.7972860381679391, "accuracy_std_dev": 0.010877999999999999, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.01688720610687023, "avg_duration_per_task_ms": 167801.82442748093, "avg_output_tokens_per_task": 6563.4376590330785, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.7972582857142857, "accuracy_std_dev": 0.005891180301683102, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014391279220779221, "avg_duration_per_task_ms": 318122.0873015873, "avg_output_tokens_per_task": 14823.277056277057, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7971204358638745, "accuracy_std_dev": 0.006421655480433002, "run_count": 8, "total_questions": 1528, "avg_cost_per_task_usd": 0.05209169568062827, "avg_duration_per_task_ms": 1143446.0909685863, "avg_output_tokens_per_task": 23739.34227748691, "last_run_timestamp": "2026-10-10T16:19:37.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Fireworks", "accuracy": 0.797043, "accuracy_std_dev": null, "run_count": 1, "total_questions": 124, "avg_cost_per_task_usd": 0.11967770967741935, "avg_duration_per_task_ms": 375576.0080645161, "avg_output_tokens_per_task": 29097.58064516129, "last_run_timestamp": "2026-07-18T08:32:17.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7968575, "accuracy_std_dev": 0.009506569460992069, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015575840067340066, "avg_duration_per_task_ms": 306607.4048821549, "avg_output_tokens_per_task": 14907.355218855218, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7966842408376964, "accuracy_std_dev": 0.016353751013826204, "run_count": 3, "total_questions": 573, "avg_cost_per_task_usd": 0.034378357766143106, "avg_duration_per_task_ms": 555130.9528795811, "avg_output_tokens_per_task": 34351.207678883075, "last_run_timestamp": "2026-10-10T16:22:09.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7964375923664123, "accuracy_std_dev": 0.11378331267012533, "run_count": 4, "total_questions": 655, "avg_cost_per_task_usd": 0.011028116030534352, "avg_duration_per_task_ms": 270893.28854961833, "avg_output_tokens_per_task": 23218.567938931297, "last_run_timestamp": "2026-09-12T08:53:48.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7961511097178684, "accuracy_std_dev": 0.09902156060353723, "run_count": 10, "total_questions": 1914, "avg_cost_per_task_usd": 0.07970391797283176, "avg_duration_per_task_ms": 762521.6421107628, "avg_output_tokens_per_task": 29044.879310344826, "last_run_timestamp": "2026-10-10T16:29:40.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7956228, "accuracy_std_dev": 0.021522407276138952, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006844225252525253, "avg_duration_per_task_ms": 363476.91515151516, "avg_output_tokens_per_task": 16652.98888888889, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.7956228, "accuracy_std_dev": 0.009072301348610534, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.051206081818181814, "avg_duration_per_task_ms": 443200.41616161616, "avg_output_tokens_per_task": 24093.476767676766, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.794007441011236, "accuracy_std_dev": 0.03030512423976348, "run_count": 4, "total_questions": 356, "avg_cost_per_task_usd": 0.07293162640449438, "avg_duration_per_task_ms": 115799.61235955056, "avg_output_tokens_per_task": 31686.629213483146, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7937475724725943, "accuracy_std_dev": 0.018702947078792137, "run_count": 9, "total_questions": 1642, "avg_cost_per_task_usd": 0.042084668696711326, "avg_duration_per_task_ms": 722711.6863580999, "avg_output_tokens_per_task": 19447.1790499391, "last_run_timestamp": "2026-10-10T16:21:30.038Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7929292857142858, "accuracy_std_dev": 0.015164807649990954, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.017839756854256855, "avg_duration_per_task_ms": 60613.86363636364, "avg_output_tokens_per_task": 5792.939393939394, "last_run_timestamp": "2026-10-10T16:11:11.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7925169999999999, "accuracy_std_dev": 0.001701000000000008, "run_count": 2, "total_questions": 392, "avg_cost_per_task_usd": 0.01095936224489796, "avg_duration_per_task_ms": 929468.150510204, "avg_output_tokens_per_task": 38601.85459183674, "last_run_timestamp": "2026-10-10T16:21:21.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7920900579268293, "accuracy_std_dev": 0.011512971285033247, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.01824498475609756, "avg_duration_per_task_ms": 181543.2469512195, "avg_output_tokens_per_task": 15051.83943089431, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7918084721906924, "accuracy_std_dev": 0.14984325869964388, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.016061496027241773, "avg_duration_per_task_ms": 528116.8059023836, "avg_output_tokens_per_task": 25582.743473325765, "last_run_timestamp": "2026-10-03T08:06:43.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.7906976325581395, "accuracy_std_dev": 0.021874000000000005, "run_count": 2, "total_questions": 215, "avg_cost_per_task_usd": 0.04791340930232558, "avg_duration_per_task_ms": 264094.9441860465, "avg_output_tokens_per_task": 20226.018604651163, "last_run_timestamp": "2026-10-10T16:22:35.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7905518785249456, "accuracy_std_dev": 0.008184215100810029, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.1894372812725958, "avg_duration_per_task_ms": 850759.7664497469, "avg_output_tokens_per_task": 47361.36731742589, "last_run_timestamp": "2026-10-10T16:22:21.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7905404425675676, "accuracy_std_dev": 0.02562269386086286, "run_count": 4, "total_questions": 592, "avg_cost_per_task_usd": 0.010724150337837838, "avg_duration_per_task_ms": 1222418.4949324324, "avg_output_tokens_per_task": 38033.55743243243, "last_run_timestamp": "2026-10-10T08:14:41.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7903146482300885, "accuracy_std_dev": 0.04010989916950373, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.14060938569321535, "avg_duration_per_task_ms": 1540325.2824483777, "avg_output_tokens_per_task": 46576.46017699115, "last_run_timestamp": "2026-10-03T16:58:27.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7902677562043796, "accuracy_std_dev": 0.015064519424931276, "run_count": 11, "total_questions": 2055, "avg_cost_per_task_usd": 0.0048938481751824815, "avg_duration_per_task_ms": 1278859.9883211679, "avg_output_tokens_per_task": 32183.068613138686, "last_run_timestamp": "2026-10-10T16:21:36.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7899609875730994, "accuracy_std_dev": 0.0876067404547278, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.12032361038011698, "avg_duration_per_task_ms": 899417.8004385965, "avg_output_tokens_per_task": 33979.93567251462, "last_run_timestamp": "2026-10-10T16:23:26.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7892815833333334, "accuracy_std_dev": 0.01859263003835273, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.004778486531986532, "avg_duration_per_task_ms": 256643.10227272726, "avg_output_tokens_per_task": 31311.578703703704, "last_run_timestamp": "2026-10-10T16:23:45.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.788807633295838, "accuracy_std_dev": 0.019769970874916862, "run_count": 9, "total_questions": 1778, "avg_cost_per_task_usd": 0.062063327896512935, "avg_duration_per_task_ms": 174797.196287964, "avg_output_tokens_per_task": 19478.599550056242, "last_run_timestamp": "2026-10-10T08:21:28.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.788428937579618, "accuracy_std_dev": 0.09448363391936922, "run_count": 8, "total_questions": 1570, "avg_cost_per_task_usd": 0.05568409044585987, "avg_duration_per_task_ms": 1688216.5210191084, "avg_output_tokens_per_task": 18918.31210191083, "last_run_timestamp": "2026-10-10T16:19:34.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.788165837398374, "accuracy_std_dev": 0.004809000000000008, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.0017273360433604334, "avg_duration_per_task_ms": 800398.4254742548, "avg_output_tokens_per_task": 11344.837398373984, "last_run_timestamp": "2026-10-10T16:23:10.694Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.78787875, "accuracy_std_dev": 0.011355865409007798, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01991093813131313, "avg_duration_per_task_ms": 480055.4835858586, "avg_output_tokens_per_task": 26459.91287878788, "last_run_timestamp": "2026-10-10T16:20:24.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7875786128564525, "accuracy_std_dev": 0.015275190978232898, "run_count": 11, "total_questions": 2069, "avg_cost_per_task_usd": 0.008747213629772837, "avg_duration_per_task_ms": 399214.60318994685, "avg_output_tokens_per_task": 34637.547124214594, "last_run_timestamp": "2026-10-10T16:21:37.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7875725454545454, "accuracy_std_dev": 0.0347531688845875, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.04297859274563819, "avg_duration_per_task_ms": 486549.3893480257, "avg_output_tokens_per_task": 42831.01193755739, "last_run_timestamp": "2026-10-10T16:19:57.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.7853823536121673, "accuracy_std_dev": 0.0034548367906458246, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.013724529784537389, "avg_duration_per_task_ms": 462082.7122940431, "avg_output_tokens_per_task": 14299.747782002534, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7839506666666667, "accuracy_std_dev": 0.054971360848677896, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0028603939393939395, "avg_duration_per_task_ms": 827726.2415824916, "avg_output_tokens_per_task": 17815.154882154882, "last_run_timestamp": "2026-10-10T16:11:11.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7835498571428571, "accuracy_std_dev": 0.056925857417292156, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.17942975324675325, "avg_duration_per_task_ms": 1414227.1096681096, "avg_output_tokens_per_task": 45704.106782106785, "last_run_timestamp": "2026-10-03T17:00:45.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.783249125, "accuracy_std_dev": 0.011716709034937033, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.021132419191919192, "avg_duration_per_task_ms": 279205.4166666667, "avg_output_tokens_per_task": 12569.390782828283, "last_run_timestamp": "2026-10-10T16:11:11.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7828282857142858, "accuracy_std_dev": 0.12584004230395515, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14693832467532467, "avg_duration_per_task_ms": 1871581.0685425685, "avg_output_tokens_per_task": 51842.28354978355, "last_run_timestamp": "2026-10-10T16:26:01.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7822926363636363, "accuracy_std_dev": 0.04547173391594292, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.19412152387511478, "avg_duration_per_task_ms": 821398.3898071626, "avg_output_tokens_per_task": 64866.6712580349, "last_run_timestamp": "2026-10-10T08:27:35.179Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7821018375634519, "accuracy_std_dev": 0.023657804732591572, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.05520209024252679, "avg_duration_per_task_ms": 511206.14438804286, "avg_output_tokens_per_task": 23994.640721940214, "last_run_timestamp": "2026-10-10T16:19:34.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7814065297191379, "accuracy_std_dev": 0.1322181089733929, "run_count": 8, "total_questions": 1531, "avg_cost_per_task_usd": 0.023490106466361852, "avg_duration_per_task_ms": 346686.11430437624, "avg_output_tokens_per_task": 23958.977139124756, "last_run_timestamp": "2026-10-10T16:20:24.759Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.781374562749004, "accuracy_std_dev": 0.09476171410543395, "run_count": 10, "total_questions": 1004, "avg_cost_per_task_usd": 0.004642227091633467, "avg_duration_per_task_ms": 236467.00996015937, "avg_output_tokens_per_task": 19258.703187250994, "last_run_timestamp": "2026-10-10T16:20:22.051Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.781145, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.020832949494949494, "avg_duration_per_task_ms": 266171.0505050505, "avg_output_tokens_per_task": 12380.515151515152, "last_run_timestamp": "2026-07-18T16:16:42.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7790441154411766, "accuracy_std_dev": 0.014485910522101155, "run_count": 9, "total_questions": 1360, "avg_cost_per_task_usd": 0.05105861102941177, "avg_duration_per_task_ms": 1000104.9169117647, "avg_output_tokens_per_task": 26002.289705882355, "last_run_timestamp": "2026-10-10T16:22:34.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7784707032418954, "accuracy_std_dev": 0.1210615547882977, "run_count": 5, "total_questions": 802, "avg_cost_per_task_usd": 0.01958750872817955, "avg_duration_per_task_ms": 560824.7643391521, "avg_output_tokens_per_task": 20557.403990024937, "last_run_timestamp": "2026-10-10T08:36:49.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7784511999999999, "accuracy_std_dev": 0.0206074889047647, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.05598715454545455, "avg_duration_per_task_ms": 643404.0878787879, "avg_output_tokens_per_task": 23899.834343434344, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7777777142857143, "accuracy_std_dev": 0.011167032598735643, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.018922173881673884, "avg_duration_per_task_ms": 673737.0562770562, "avg_output_tokens_per_task": 47091.12914862915, "last_run_timestamp": "2026-10-10T16:25:17.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7774715454545453, "accuracy_std_dev": 0.019178897604889684, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.005792984389348025, "avg_duration_per_task_ms": 704409.8140495868, "avg_output_tokens_per_task": 36399.0564738292, "last_run_timestamp": "2026-10-10T16:20:12.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7772166666666667, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013080006734006736, "avg_duration_per_task_ms": 67697.39225589225, "avg_output_tokens_per_task": 10341.513468013469, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.777134539797395, "accuracy_std_dev": 0.02043543185504141, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.022169380607814763, "avg_duration_per_task_ms": 860418.1266280753, "avg_output_tokens_per_task": 38276.162807525325, "last_run_timestamp": "2026-10-10T16:27:48.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.77567325, "accuracy_std_dev": 0.010934638822910419, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013401992424242422, "avg_duration_per_task_ms": 67621.48106060606, "avg_output_tokens_per_task": 10598.125, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7741139703903096, "accuracy_std_dev": 0.09718478875928833, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.004475604306864065, "avg_duration_per_task_ms": 312870.95827725437, "avg_output_tokens_per_task": 22926.07267833109, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7739002377332412, "accuracy_std_dev": 0.03919556245291041, "run_count": 9, "total_questions": 1447, "avg_cost_per_task_usd": 0.05904209744298548, "avg_duration_per_task_ms": 844538.2218382861, "avg_output_tokens_per_task": 50153.10711817553, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7737795, "accuracy_std_dev": 0.0353227557921236, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05213979987373737, "avg_duration_per_task_ms": 865410.4703282828, "avg_output_tokens_per_task": 23453.041035353537, "last_run_timestamp": "2026-10-10T16:19:57.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.7736572813299233, "accuracy_std_dev": 0.015466646781596444, "run_count": 3, "total_questions": 391, "avg_cost_per_task_usd": 0.0087913273657289, "avg_duration_per_task_ms": 647179.7749360614, "avg_output_tokens_per_task": 29178.061381074167, "last_run_timestamp": "2026-10-10T16:25:53.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7730078333333333, "accuracy_std_dev": 0.02074231892706846, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015851826599326598, "avg_duration_per_task_ms": 424890.48232323234, "avg_output_tokens_per_task": 15979.028619528619, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7728757843137255, "accuracy_std_dev": 0.0424837728239854, "run_count": 5, "total_questions": 714, "avg_cost_per_task_usd": 0.13942839355742298, "avg_duration_per_task_ms": 1052327.7044817926, "avg_output_tokens_per_task": 40065.01820728291, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7722222999999999, "accuracy_std_dev": 0.010833447964983244, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.014700993434343435, "avg_duration_per_task_ms": 139170.2106060606, "avg_output_tokens_per_task": 29356.293434343435, "last_run_timestamp": "2026-10-10T16:22:35.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7715388278781039, "accuracy_std_dev": 0.007999827803702316, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.011440101015801354, "avg_duration_per_task_ms": 999816.4932279909, "avg_output_tokens_per_task": 11904.952031602708, "last_run_timestamp": "2026-10-10T16:20:24.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7710935525846703, "accuracy_std_dev": 0.018076003519091888, "run_count": 6, "total_questions": 1122, "avg_cost_per_task_usd": 0.0213185614973262, "avg_duration_per_task_ms": 243272.5650623886, "avg_output_tokens_per_task": 28339.197860962566, "last_run_timestamp": "2026-10-10T16:21:03.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7704125, "accuracy_std_dev": 0.011714588949254664, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005121838383838384, "avg_duration_per_task_ms": 1054586.8964646466, "avg_output_tokens_per_task": 28369.739898989897, "last_run_timestamp": "2026-10-10T16:19:34.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.770202, "accuracy_std_dev": 0.008625232557251217, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013833271043771043, "avg_duration_per_task_ms": 119359.86616161616, "avg_output_tokens_per_task": 10422.487373737375, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7701042403314916, "accuracy_std_dev": 0.009818378139451328, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.018828466850828732, "avg_duration_per_task_ms": 104392.93784530387, "avg_output_tokens_per_task": 5944.6809392265195, "last_run_timestamp": "2026-10-10T08:22:24.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7693388734177216, "accuracy_std_dev": 0.018315649062573065, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.058541679324894506, "avg_duration_per_task_ms": 602157.2911392405, "avg_output_tokens_per_task": 26894.77299578059, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7692142947976879, "accuracy_std_dev": 0.049901425655830466, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.2430246191393706, "avg_duration_per_task_ms": 900127.0802825948, "avg_output_tokens_per_task": 70448.42453436095, "last_run_timestamp": "2026-10-10T16:19:34.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7691335028462997, "accuracy_std_dev": 0.0413125157602088, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.14881955850727388, "avg_duration_per_task_ms": 1030577.8734977862, "avg_output_tokens_per_task": 67777.00316255534, "last_run_timestamp": "2026-10-10T16:11:14.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.768254, "accuracy_std_dev": null, "run_count": 1, "total_questions": 105, "avg_cost_per_task_usd": 0.04709281904761905, "avg_duration_per_task_ms": 1085275.6761904762, "avg_output_tokens_per_task": 21896.628571428573, "last_run_timestamp": "2026-07-18T16:43:10.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7682437087198516, "accuracy_std_dev": 0.017109400315529023, "run_count": 11, "total_questions": 2156, "avg_cost_per_task_usd": 0.00780624628942486, "avg_duration_per_task_ms": 455549.38729128015, "avg_output_tokens_per_task": 27461.665584415583, "last_run_timestamp": "2026-10-10T16:19:40.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7682173900573613, "accuracy_std_dev": 0.06513957389797216, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.260235, "avg_duration_per_task_ms": 959583.7954110899, "avg_output_tokens_per_task": 74013.42192479286, "last_run_timestamp": "2026-10-10T16:11:11.315Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7680976249999999, "accuracy_std_dev": 0.03807231169740517, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1701674728535354, "avg_duration_per_task_ms": 546025.8289141414, "avg_output_tokens_per_task": 52982.59659090909, "last_run_timestamp": "2026-10-10T16:20:22.051Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7671156666666666, "accuracy_std_dev": 0.03093018545405481, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013063415824915825, "avg_duration_per_task_ms": 936092.1515151515, "avg_output_tokens_per_task": 46359.02693602694, "last_run_timestamp": "2026-10-10T08:15:34.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7665065498199279, "accuracy_std_dev": 0.0370963104843463, "run_count": 7, "total_questions": 833, "avg_cost_per_task_usd": 0.012696330132052819, "avg_duration_per_task_ms": 53169.84753901561, "avg_output_tokens_per_task": 16839.417767106843, "last_run_timestamp": "2026-10-10T16:20:08.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.7659934999999999, "accuracy_std_dev": 0.008417499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007677666666666667, "avg_duration_per_task_ms": 154673.01515151514, "avg_output_tokens_per_task": 30515.89393939394, "last_run_timestamp": "2026-10-10T16:21:31.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.765993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08016306060606061, "avg_duration_per_task_ms": 425226.22222222225, "avg_output_tokens_per_task": 69822.58585858585, "last_run_timestamp": "2026-07-18T08:29:23.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7652717142857143, "accuracy_std_dev": 0.015187772344077012, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.019384612554112553, "avg_duration_per_task_ms": 1111853.9927849928, "avg_output_tokens_per_task": 48423.183982683986, "last_run_timestamp": "2026-10-10T16:21:25.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7650936065232479, "accuracy_std_dev": 0.10998854890673562, "run_count": 8, "total_questions": 1441, "avg_cost_per_task_usd": 0.036296903539208884, "avg_duration_per_task_ms": 147941.55586398335, "avg_output_tokens_per_task": 29262.30326162387, "last_run_timestamp": "2026-10-10T16:22:02.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7638156348684211, "accuracy_std_dev": 0.006318496220175736, "run_count": 8, "total_questions": 1520, "avg_cost_per_task_usd": 0.024435126315789476, "avg_duration_per_task_ms": 70438.73486842106, "avg_output_tokens_per_task": 25588.723684210527, "last_run_timestamp": "2026-10-10T16:21:31.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7630074302030456, "accuracy_std_dev": 0.05691748932171465, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.007098315989847716, "avg_duration_per_task_ms": 964817.2912436548, "avg_output_tokens_per_task": 18426.44416243655, "last_run_timestamp": "2026-10-10T16:11:11.298Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7613855279025016, "accuracy_std_dev": 0.0736067708315614, "run_count": 8, "total_questions": 1559, "avg_cost_per_task_usd": 0.24665575304682488, "avg_duration_per_task_ms": 1221066.671584349, "avg_output_tokens_per_task": 78685.70686337395, "last_run_timestamp": "2026-10-10T16:19:34.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7611530000000001, "accuracy_std_dev": 0.04936588290509955, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01190072095959596, "avg_duration_per_task_ms": 817020.8775252525, "avg_output_tokens_per_task": 36142.90782828283, "last_run_timestamp": "2026-10-10T08:14:45.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.760943, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08179280303030302, "avg_duration_per_task_ms": 426340.89898989897, "avg_output_tokens_per_task": 70994.31818181818, "last_run_timestamp": "2026-07-18T08:26:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7605754720812183, "accuracy_std_dev": 0.010447569992586773, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0025078527918781725, "avg_duration_per_task_ms": 100160.06345177664, "avg_output_tokens_per_task": 31388.34771573604, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.7605443265306122, "accuracy_std_dev": 0.0343317834601635, "run_count": 3, "total_questions": 490, "avg_cost_per_task_usd": 0.0037374408163265305, "avg_duration_per_task_ms": 668153.0571428571, "avg_output_tokens_per_task": 14206.583673469388, "last_run_timestamp": "2026-08-08T11:42:26.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7604232514285715, "accuracy_std_dev": 0.03409113144550026, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.11787924444444446, "avg_duration_per_task_ms": 816938.3003174603, "avg_output_tokens_per_task": 46021.84, "last_run_timestamp": "2026-10-10T16:11:11.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7596333333333334, "accuracy_std_dev": 0.09860502035562557, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03538440628507295, "avg_duration_per_task_ms": 1290567.9764309765, "avg_output_tokens_per_task": 20956.203703703704, "last_run_timestamp": "2026-10-10T16:22:34.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.75925925, "accuracy_std_dev": 0.006627946132664344, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017700304292929292, "avg_duration_per_task_ms": 80251.25441919192, "avg_output_tokens_per_task": 29259.58712121212, "last_run_timestamp": "2026-10-10T16:22:35.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7590111668383109, "accuracy_std_dev": 0.016611473377157127, "run_count": 5, "total_questions": 971, "avg_cost_per_task_usd": 0.004071515962924819, "avg_duration_per_task_ms": 4054.4541709577757, "avg_output_tokens_per_task": 2252.576725025747, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7586981111111112, "accuracy_std_dev": 0.018373432640300367, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.016781338945005612, "avg_duration_per_task_ms": 1197308.150392817, "avg_output_tokens_per_task": 51632.826599326596, "last_run_timestamp": "2026-10-10T16:22:45.582Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.758698, "accuracy_std_dev": 0.012009662443216293, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.056970961279461274, "avg_duration_per_task_ms": 99388.35016835017, "avg_output_tokens_per_task": 22699.07239057239, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7586279375000001, "accuracy_std_dev": 0.013868153578742699, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.01862196054292929, "avg_duration_per_task_ms": 165357.07323232322, "avg_output_tokens_per_task": 30815.23611111111, "last_run_timestamp": "2026-10-10T16:25:17.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7585079293193718, "accuracy_std_dev": 0.0038942041789690614, "run_count": 4, "total_questions": 764, "avg_cost_per_task_usd": 0.01819221335078534, "avg_duration_per_task_ms": 1032018.7041884817, "avg_output_tokens_per_task": 55015.78272251309, "last_run_timestamp": "2026-10-10T16:22:44.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.757365375, "accuracy_std_dev": 0.009939804285516652, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.018617486111111112, "avg_duration_per_task_ms": 159086.00441919192, "avg_output_tokens_per_task": 30836.404671717173, "last_run_timestamp": "2026-10-10T16:19:34.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7569024000000001, "accuracy_std_dev": 0.0779853723171211, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.017876224242424243, "avg_duration_per_task_ms": 1933669.407070707, "avg_output_tokens_per_task": 17409.9303030303, "last_run_timestamp": "2026-10-10T16:21:15.010Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.7565450000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.020916916230366494, "avg_duration_per_task_ms": 124793.11518324607, "avg_output_tokens_per_task": 18115.47120418848, "last_run_timestamp": "2026-07-18T08:26:49.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7561492633587785, "accuracy_std_dev": 0.013604073603497425, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.01678382569974555, "avg_duration_per_task_ms": 286656.0801526718, "avg_output_tokens_per_task": 28394.44720101781, "last_run_timestamp": "2026-10-10T16:19:40.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7558386418966737, "accuracy_std_dev": 0.03258665299412283, "run_count": 8, "total_questions": 1413, "avg_cost_per_task_usd": 0.007369956829440905, "avg_duration_per_task_ms": 1050257.7034677991, "avg_output_tokens_per_task": 18996.81599433829, "last_run_timestamp": "2026-10-10T16:10:48.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7556517142857143, "accuracy_std_dev": 0.007605047378725084, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.022500825396825398, "avg_duration_per_task_ms": 97229.08225108225, "avg_output_tokens_per_task": 14839.808080808081, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7553378326797385, "accuracy_std_dev": 0.026370845649125493, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.015925873202614377, "avg_duration_per_task_ms": 239486.98169934642, "avg_output_tokens_per_task": 49403.856209150326, "last_run_timestamp": "2026-10-10T08:36:35.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7552047101754386, "accuracy_std_dev": 0.036034774046985296, "run_count": 9, "total_questions": 1425, "avg_cost_per_task_usd": 0.04509595859649123, "avg_duration_per_task_ms": 263424.8596491228, "avg_output_tokens_per_task": 16991.12350877193, "last_run_timestamp": "2026-10-10T08:22:13.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.75515575, "accuracy_std_dev": 0.06516829345385299, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1108029071969697, "avg_duration_per_task_ms": 571979.9936868687, "avg_output_tokens_per_task": 33259.57323232323, "last_run_timestamp": "2026-10-10T16:21:36.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7535393918918919, "accuracy_std_dev": 0.07402326310019032, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.13893272265122267, "avg_duration_per_task_ms": 1010369.6351351351, "avg_output_tokens_per_task": 58604.81145431146, "last_run_timestamp": "2026-10-10T16:10:32.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.753367, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015499042929292928, "avg_duration_per_task_ms": 1045731.0252525252, "avg_output_tokens_per_task": 55053.381313131315, "last_run_timestamp": "2026-10-10T08:14:49.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7524220592034444, "accuracy_std_dev": 0.028639192749568526, "run_count": 7, "total_questions": 929, "avg_cost_per_task_usd": 0.05453179763186222, "avg_duration_per_task_ms": 423386.32615715824, "avg_output_tokens_per_task": 15836.034445640473, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7509663565217392, "accuracy_std_dev": 0.031143314111326263, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.0046609500000000005, "avg_duration_per_task_ms": 782901.7971014492, "avg_output_tokens_per_task": 28031.711594202898, "last_run_timestamp": "2026-10-10T16:21:18.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.750842, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016549714646464646, "avg_duration_per_task_ms": 972340.9924242424, "avg_output_tokens_per_task": 58750.5101010101, "last_run_timestamp": "2026-10-10T08:22:25.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.75076231097561, "accuracy_std_dev": 0.10261490128027215, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.006507009654471546, "avg_duration_per_task_ms": 566183.218495935, "avg_output_tokens_per_task": 27810.678353658535, "last_run_timestamp": "2026-10-10T08:22:53.947Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.749368625, "accuracy_std_dev": 0.008643231874384451, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.013557867424242424, "avg_duration_per_task_ms": 96888.43560606061, "avg_output_tokens_per_task": 14901.809343434343, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.748737375, "accuracy_std_dev": 0.14136326117306566, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02991949936868687, "avg_duration_per_task_ms": 136944.42866161617, "avg_output_tokens_per_task": 22918.396464646463, "last_run_timestamp": "2026-10-10T16:20:08.973Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.748737375, "accuracy_std_dev": 0.03686209625054409, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.052284164141414145, "avg_duration_per_task_ms": 944565.4507575758, "avg_output_tokens_per_task": 27129.21275252525, "last_run_timestamp": "2026-10-10T16:23:01.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.7481442884615385, "accuracy_std_dev": 0.0227081467548543, "run_count": 5, "total_questions": 988, "avg_cost_per_task_usd": 0.020536520242914982, "avg_duration_per_task_ms": 151472.34615384616, "avg_output_tokens_per_task": 11878.324898785426, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.7481013645569621, "accuracy_std_dev": 0.004003499999999993, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.010130237974683544, "avg_duration_per_task_ms": 298779.6734177215, "avg_output_tokens_per_task": 60721.07341772152, "last_run_timestamp": "2026-10-10T08:39:12.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.747826043478261, "accuracy_std_dev": 0.06491713177676944, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.010265418115942029, "avg_duration_per_task_ms": 524781.3072463768, "avg_output_tokens_per_task": 34823.62463768116, "last_run_timestamp": "2026-10-10T16:20:00.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7478116, "accuracy_std_dev": 0.02151720496347049, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08178134747474748, "avg_duration_per_task_ms": 885057.2202020203, "avg_output_tokens_per_task": 23371.92828282828, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08105034848484848, "avg_duration_per_task_ms": 422601, "avg_output_tokens_per_task": 70367.68181818182, "last_run_timestamp": "2026-07-18T08:27:39.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7471013608695652, "accuracy_std_dev": 0.06255498618570607, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.2582489231884058, "avg_duration_per_task_ms": 1276224.2644927537, "avg_output_tokens_per_task": 82450.74855072463, "last_run_timestamp": "2026-10-10T16:20:06.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7470487959527824, "accuracy_std_dev": 0.08033451694532995, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.005070625070264193, "avg_duration_per_task_ms": 656330.5952782462, "avg_output_tokens_per_task": 33326.38673412029, "last_run_timestamp": "2026-09-26T18:04:35.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.746633, "accuracy_std_dev": 0.017677, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017912724747474747, "avg_duration_per_task_ms": 70698.09595959596, "avg_output_tokens_per_task": 6119.75, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.74642275, "accuracy_std_dev": 0.008945525344411051, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01488374558080808, "avg_duration_per_task_ms": 107961.75883838384, "avg_output_tokens_per_task": 14400.517676767677, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7461957648578811, "accuracy_std_dev": 0.05992671984854132, "run_count": 6, "total_questions": 1161, "avg_cost_per_task_usd": 0.1759588509905254, "avg_duration_per_task_ms": 1560060.6244616709, "avg_output_tokens_per_task": 42375.20930232558, "last_run_timestamp": "2026-10-10T16:23:46.181Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7459782222222222, "accuracy_std_dev": 0.009438398095931553, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006886203142536476, "avg_duration_per_task_ms": 163678.69079685747, "avg_output_tokens_per_task": 8003.4517396184065, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7454796043395022, "accuracy_std_dev": 0.1116514794950676, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.16252882514358646, "avg_duration_per_task_ms": 1066174.2463305679, "avg_output_tokens_per_task": 46648.48819400128, "last_run_timestamp": "2026-10-10T08:21:42.283Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7434143503039515, "accuracy_std_dev": 0.004376274946591358, "run_count": 7, "total_questions": 1316, "avg_cost_per_task_usd": 0.009106649696048632, "avg_duration_per_task_ms": 140201.952887538, "avg_output_tokens_per_task": 11539.308510638299, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.743266, "accuracy_std_dev": 0.006355106254029135, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005372063762626263, "avg_duration_per_task_ms": 150638.75315656565, "avg_output_tokens_per_task": 8682.62563131313, "last_run_timestamp": "2026-10-10T16:21:31.113Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.743145857142857, "accuracy_std_dev": 0.011088924595914524, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.023026453102453104, "avg_duration_per_task_ms": 99585.75685425685, "avg_output_tokens_per_task": 15242.642857142857, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7415824999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.019016893939393938, "avg_duration_per_task_ms": 33403.05808080808, "avg_output_tokens_per_task": 6109.055555555556, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.740951125, "accuracy_std_dev": 0.06026863513561074, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01223913446969697, "avg_duration_per_task_ms": 98981.7702020202, "avg_output_tokens_per_task": 5413.777777777777, "last_run_timestamp": "2026-10-10T08:14:41.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.740741, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02792859090909091, "avg_duration_per_task_ms": 682232.3585858586, "avg_output_tokens_per_task": 23364.934343434343, "last_run_timestamp": "2026-07-18T09:15:42.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.739899, "accuracy_std_dev": 0.010812416138403098, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0048791994949494944, "avg_duration_per_task_ms": 108739.66098484848, "avg_output_tokens_per_task": 7956.729797979798, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.7392292244897959, "accuracy_std_dev": 0.110097, "run_count": 2, "total_questions": 294, "avg_cost_per_task_usd": 0.03822905782312925, "avg_duration_per_task_ms": 1403982.7517006802, "avg_output_tokens_per_task": 27800.656462585033, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7390017868020304, "accuracy_std_dev": 0.059270287748631335, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.01651792470389171, "avg_duration_per_task_ms": 1126017.0372250422, "avg_output_tokens_per_task": 41901.6192893401, "last_run_timestamp": "2026-10-10T16:29:23.083Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.738906892857143, "accuracy_std_dev": 0.016676086646452728, "run_count": 5, "total_questions": 616, "avg_cost_per_task_usd": 0.015153974025974027, "avg_duration_per_task_ms": 72126.25162337662, "avg_output_tokens_per_task": 8710.556818181818, "last_run_timestamp": "2026-10-10T16:10:51.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.738309, "accuracy_std_dev": 0.008136215650479735, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.059655349046015715, "avg_duration_per_task_ms": 125601.6632996633, "avg_output_tokens_per_task": 23853.966891133558, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7373739999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.020083924242424243, "avg_duration_per_task_ms": 186579.25757575757, "avg_output_tokens_per_task": 62586.09595959596, "last_run_timestamp": "2026-10-10T16:27:29.137Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.736111, "accuracy_std_dev": null, "run_count": 1, "total_questions": 168, "avg_cost_per_task_usd": 0.05987852380952381, "avg_duration_per_task_ms": 1235915.3511904762, "avg_output_tokens_per_task": 62102.07738095238, "last_run_timestamp": "2026-07-18T08:29:05.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7356996188835285, "accuracy_std_dev": 0.006961170963817794, "run_count": 8, "total_questions": 1451, "avg_cost_per_task_usd": 0.00648628256374914, "avg_duration_per_task_ms": 228531.34045485873, "avg_output_tokens_per_task": 8024.518952446589, "last_run_timestamp": "2026-10-10T16:22:35.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7348483749999999, "accuracy_std_dev": 0.07315972270644808, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006649549242424242, "avg_duration_per_task_ms": 101923.77272727272, "avg_output_tokens_per_task": 16353.064393939394, "last_run_timestamp": "2026-10-10T16:19:40.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7343453415559772, "accuracy_std_dev": 0.11879281780682018, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.005205407969639469, "avg_duration_per_task_ms": 325171.4794433903, "avg_output_tokens_per_task": 14219.585072738773, "last_run_timestamp": "2026-10-10T16:11:11.296Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.73327025, "accuracy_std_dev": 0.13440860173157632, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01872256628787879, "avg_duration_per_task_ms": 698828.0744949495, "avg_output_tokens_per_task": 35401.32323232323, "last_run_timestamp": "2026-10-10T08:33:01.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.731691875, "accuracy_std_dev": 0.013065716091526514, "run_count": 32, "total_questions": 6336, "avg_cost_per_task_usd": 0.2044065787563131, "avg_duration_per_task_ms": 243404.04545454544, "avg_output_tokens_per_task": 38177.99210858586, "last_run_timestamp": "2026-10-10T08:24:42.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7310988181818183, "accuracy_std_dev": 0.0161010559157872, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1857441469237833, "avg_duration_per_task_ms": 294606.93709825526, "avg_output_tokens_per_task": 36982.24701561065, "last_run_timestamp": "2026-10-10T08:24:31.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7309332329663789, "accuracy_std_dev": 0.016291390037575088, "run_count": 17, "total_questions": 3361, "avg_cost_per_task_usd": 0.0067620446295745314, "avg_duration_per_task_ms": 112562.21005653079, "avg_output_tokens_per_task": 7053.808985421006, "last_run_timestamp": "2026-10-10T16:26:34.195Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.730219, "accuracy_std_dev": 0.02007444496742065, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005502738005050506, "avg_duration_per_task_ms": 219251.19823232322, "avg_output_tokens_per_task": 7924.819444444444, "last_run_timestamp": "2026-10-10T16:20:00.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7301589081632653, "accuracy_std_dev": 0.01053611207018773, "run_count": 3, "total_questions": 588, "avg_cost_per_task_usd": 0.01497738095238095, "avg_duration_per_task_ms": 995236.0867346938, "avg_output_tokens_per_task": 53138.4693877551, "last_run_timestamp": "2026-10-10T08:23:30.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7284971818181819, "accuracy_std_dev": 0.009660041669571141, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.19039750918273646, "avg_duration_per_task_ms": 242398.78512396695, "avg_output_tokens_per_task": 37888.70890725436, "last_run_timestamp": "2026-10-10T08:14:40.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7277537142857143, "accuracy_std_dev": 0.011584046612404796, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006112832611832612, "avg_duration_per_task_ms": 377298.1096681097, "avg_output_tokens_per_task": 8091.412698412699, "last_run_timestamp": "2026-10-10T16:21:31.197Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7274832499999999, "accuracy_std_dev": 0.02903040114151885, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007206551136363636, "avg_duration_per_task_ms": 297296.3061868687, "avg_output_tokens_per_task": 17913.31376262626, "last_run_timestamp": "2026-10-10T16:19:34.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7272725714285714, "accuracy_std_dev": 0.015455781912624552, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004581955266955267, "avg_duration_per_task_ms": 160113.02813852814, "avg_output_tokens_per_task": 7774.143578643579, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7271808509532062, "accuracy_std_dev": 0.03540222199908109, "run_count": 6, "total_questions": 1154, "avg_cost_per_task_usd": 0.011152869150779895, "avg_duration_per_task_ms": 504501.5831889082, "avg_output_tokens_per_task": 55506.62998266898, "last_run_timestamp": "2026-10-10T08:24:27.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7261503333333333, "accuracy_std_dev": 0.004418546090690373, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.059926168350168355, "avg_duration_per_task_ms": 123042.70033670033, "avg_output_tokens_per_task": 23940.483164983165, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7258268816793894, "accuracy_std_dev": 0.021662280079307778, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.006591062340966921, "avg_duration_per_task_ms": 353326.7900763359, "avg_output_tokens_per_task": 26171.935114503816, "last_run_timestamp": "2026-10-10T16:19:40.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7243107387218045, "accuracy_std_dev": 0.009277815175446843, "run_count": 11, "total_questions": 2128, "avg_cost_per_task_usd": 0.18669253665413532, "avg_duration_per_task_ms": 347016.6790413534, "avg_output_tokens_per_task": 37148.297462406015, "last_run_timestamp": "2026-10-10T08:14:48.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.723064, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01875409595959596, "avg_duration_per_task_ms": 273477.0126262626, "avg_output_tokens_per_task": 66626.44696969698, "last_run_timestamp": "2026-10-10T08:47:12.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7229704444444446, "accuracy_std_dev": 0.011078144932063405, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.004416624017957351, "avg_duration_per_task_ms": 624923.5415263749, "avg_output_tokens_per_task": 7906.3456790123455, "last_run_timestamp": "2026-10-10T16:21:49.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7219672121212123, "accuracy_std_dev": 0.013487941678127918, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.19738882996633, "avg_duration_per_task_ms": 265847.20844811754, "avg_output_tokens_per_task": 36814.46908478727, "last_run_timestamp": "2026-10-10T08:31:35.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7189313475609755, "accuracy_std_dev": 0.11211128771311958, "run_count": 7, "total_questions": 1148, "avg_cost_per_task_usd": 0.004322345818815331, "avg_duration_per_task_ms": 557688.2717770035, "avg_output_tokens_per_task": 24194.70644599303, "last_run_timestamp": "2026-10-10T08:48:56.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7184945714285713, "accuracy_std_dev": 0.04791285835722166, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015674489177489176, "avg_duration_per_task_ms": 909238.3787878788, "avg_output_tokens_per_task": 44831.91558441558, "last_run_timestamp": "2026-10-10T16:20:06.461Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7184345000000002, "accuracy_std_dev": 0.00575544305418794, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006676434343434343, "avg_duration_per_task_ms": 1198975.5896464647, "avg_output_tokens_per_task": 54074.54671717172, "last_run_timestamp": "2026-09-26T17:35:27.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.7184345000000001, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005931222222222222, "avg_duration_per_task_ms": 277051.3181818182, "avg_output_tokens_per_task": 7213.616161616162, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7171302234809476, "accuracy_std_dev": 0.05915089253318348, "run_count": 6, "total_questions": 971, "avg_cost_per_task_usd": 0.035917489186405756, "avg_duration_per_task_ms": 751381.8146240988, "avg_output_tokens_per_task": 38973.5623069001, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.715769, "accuracy_std_dev": 0.047798870551509894, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00680639898989899, "avg_duration_per_task_ms": 127099.18602693603, "avg_output_tokens_per_task": 7554.406565656565, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.715488, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0037085252525252527, "avg_duration_per_task_ms": 839996.601010101, "avg_output_tokens_per_task": 28561.196969696968, "last_run_timestamp": "2026-07-18T16:28:34.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.7153070167364017, "accuracy_std_dev": 0.05075792219782052, "run_count": 5, "total_questions": 956, "avg_cost_per_task_usd": 0.06210530125523012, "avg_duration_per_task_ms": 1717196.740585774, "avg_output_tokens_per_task": 33817.89644351464, "last_run_timestamp": "2026-10-10T16:11:11.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7133716737044147, "accuracy_std_dev": 0.07918700388461619, "run_count": 3, "total_questions": 521, "avg_cost_per_task_usd": 0.07439642610364683, "avg_duration_per_task_ms": 652167.8637236084, "avg_output_tokens_per_task": 28734.940499040305, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.7131104709511569, "accuracy_std_dev": 0.0367116985872351, "run_count": 10, "total_questions": 1945, "avg_cost_per_task_usd": 0.010703536246786632, "avg_duration_per_task_ms": 271949.64678663237, "avg_output_tokens_per_task": 47587.55115681234, "last_run_timestamp": "2026-10-10T08:14:46.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7118807142857142, "accuracy_std_dev": 0.008414141170235586, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011084775613275614, "avg_duration_per_task_ms": 94339.78354978355, "avg_output_tokens_per_task": 10485.185425685426, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.71170025, "accuracy_std_dev": 0.018456257615982147, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0649322058080808, "avg_duration_per_task_ms": 1066951.6603535353, "avg_output_tokens_per_task": 21770.531565656565, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7113995714285716, "accuracy_std_dev": 0.010488730194930244, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007985493506493507, "avg_duration_per_task_ms": 77746.15223665224, "avg_output_tokens_per_task": 10141.235930735931, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7109188571428572, "accuracy_std_dev": 0.009120000930898651, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008814419191919191, "avg_duration_per_task_ms": 134404.79509379508, "avg_output_tokens_per_task": 5713.798701298701, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7106782857142858, "accuracy_std_dev": 0.008509694097478088, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015819831890331888, "avg_duration_per_task_ms": 94996.398989899, "avg_output_tokens_per_task": 10465.311688311689, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.7084164392026578, "accuracy_std_dev": 0.2593601641821839, "run_count": 8, "total_questions": 1505, "avg_cost_per_task_usd": 0.025249661129568106, "avg_duration_per_task_ms": 375367.3375415282, "avg_output_tokens_per_task": 15131.079734219269, "last_run_timestamp": "2026-10-10T16:19:34.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7080899875730994, "accuracy_std_dev": 0.01572159476124274, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.010683734649122806, "avg_duration_per_task_ms": 102665.67105263157, "avg_output_tokens_per_task": 9845.541666666666, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7051465714285715, "accuracy_std_dev": 0.008927717479163138, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006021121212121212, "avg_duration_per_task_ms": 109220.25974025975, "avg_output_tokens_per_task": 5588.1832611832615, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.70328275, "accuracy_std_dev": 0.02153875190644758, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0019388005050505052, "avg_duration_per_task_ms": 636092.8409090909, "avg_output_tokens_per_task": 14000.35101010101, "last_run_timestamp": "2026-09-05T08:28:07.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7024286137142859, "accuracy_std_dev": 0.04779753153655171, "run_count": 18, "total_questions": 3500, "avg_cost_per_task_usd": 0.008325981428571428, "avg_duration_per_task_ms": 489576.588, "avg_output_tokens_per_task": 25239.30742857143, "last_run_timestamp": "2026-10-10T16:21:31.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.701832040429564, "accuracy_std_dev": 0.009254626092359155, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.010471590018951359, "avg_duration_per_task_ms": 144460.25710675932, "avg_output_tokens_per_task": 9453.23373341756, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7017798571428572, "accuracy_std_dev": 0.013719199460490942, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011180141414141413, "avg_duration_per_task_ms": 199426.63852813852, "avg_output_tokens_per_task": 5820.556277056277, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.7010584285714286, "accuracy_std_dev": 0.024870525026666448, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011166472582972582, "avg_duration_per_task_ms": 61418.856421356424, "avg_output_tokens_per_task": 9244.735930735931, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6984750588235294, "accuracy_std_dev": 0.028411848992761817, "run_count": 8, "total_questions": 1530, "avg_cost_per_task_usd": 0.009441421568627451, "avg_duration_per_task_ms": 385390.5339869281, "avg_output_tokens_per_task": 44748.05424836601, "last_run_timestamp": "2026-10-10T08:22:04.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.69633825, "accuracy_std_dev": 0.11651475266972633, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.015558469065656567, "avg_duration_per_task_ms": 463991.38194444444, "avg_output_tokens_per_task": 14784.056186868687, "last_run_timestamp": "2026-10-10T16:10:48.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.6941227451737451, "accuracy_std_dev": 0.08714860354302875, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.13150269498069497, "avg_duration_per_task_ms": 1023131.1537966538, "avg_output_tokens_per_task": 71194.34684684685, "last_run_timestamp": "2026-10-10T16:10:46.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.6928812857142858, "accuracy_std_dev": 0.0184605601038251, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.009158278499278501, "avg_duration_per_task_ms": 267700.48124098126, "avg_output_tokens_per_task": 6402.472582972583, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.692129625, "accuracy_std_dev": 0.010010686751385976, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005500493055555555, "avg_duration_per_task_ms": 380445.4021464646, "avg_output_tokens_per_task": 6066.023358585859, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Wafer", "accuracy": 0.691919, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01248349494949495, "avg_duration_per_task_ms": 400806.7474747475, "avg_output_tokens_per_task": 31001.929292929293, "last_run_timestamp": "2026-07-18T08:04:42.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.691462222392638, "accuracy_std_dev": 0.028807852029534505, "run_count": 6, "total_questions": 652, "avg_cost_per_task_usd": 0.00933088036809816, "avg_duration_per_task_ms": 52547.14110429448, "avg_output_tokens_per_task": 6038.897239263803, "last_run_timestamp": "2026-10-10T16:11:11.293Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6873114230769231, "accuracy_std_dev": 0.04923369976869075, "run_count": 9, "total_questions": 1768, "avg_cost_per_task_usd": 0.007741963800904979, "avg_duration_per_task_ms": 279050.26527149323, "avg_output_tokens_per_task": 40832.55995475113, "last_run_timestamp": "2026-10-10T08:15:00.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.6856376949064449, "accuracy_std_dev": 0.10684923261228746, "run_count": 11, "total_questions": 1924, "avg_cost_per_task_usd": 0.031270327962577955, "avg_duration_per_task_ms": 1092631.9272349272, "avg_output_tokens_per_task": 32831.02286902287, "last_run_timestamp": "2026-10-10T16:21:35.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.685185, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007298616161616161, "avg_duration_per_task_ms": 578052.9191919192, "avg_output_tokens_per_task": 17923.358585858587, "last_run_timestamp": "2026-07-18T08:44:31.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.685185, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06544819696969698, "avg_duration_per_task_ms": 945099.904040404, "avg_output_tokens_per_task": 33006.14646464647, "last_run_timestamp": "2026-07-18T16:15:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.6809765714285714, "accuracy_std_dev": 0.0445115533536989, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014742932178932178, "avg_duration_per_task_ms": 268442.60533910536, "avg_output_tokens_per_task": 7991.235930735931, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6782486254237288, "accuracy_std_dev": 0.021428719439237503, "run_count": 6, "total_questions": 590, "avg_cost_per_task_usd": 0.004103340677966102, "avg_duration_per_task_ms": 494419.3559322034, "avg_output_tokens_per_task": 27366.218644067798, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6779694919852034, "accuracy_std_dev": 0.0848101181666433, "run_count": 5, "total_questions": 811, "avg_cost_per_task_usd": 0.007035279901356351, "avg_duration_per_task_ms": 476700.88409371144, "avg_output_tokens_per_task": 34252.66584463625, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6774892857142858, "accuracy_std_dev": 0.07499128841922582, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00669294227994228, "avg_duration_per_task_ms": 714224.0743145744, "avg_output_tokens_per_task": 23217.833333333332, "last_run_timestamp": "2026-10-10T16:24:54.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.676784615514334, "accuracy_std_dev": 0.11674700703943836, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.05877558516020236, "avg_duration_per_task_ms": 524798.5919055649, "avg_output_tokens_per_task": 24486.29679595278, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6750842, "accuracy_std_dev": 0.023641210809939493, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005652539393939394, "avg_duration_per_task_ms": 500294.62828282826, "avg_output_tokens_per_task": 16033.422222222222, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6744752254758419, "accuracy_std_dev": 0.018240929771167532, "run_count": 6, "total_questions": 683, "avg_cost_per_task_usd": 0.004128366032210834, "avg_duration_per_task_ms": 765372.7452415812, "avg_output_tokens_per_task": 41143.20790629576, "last_run_timestamp": "2026-10-10T16:22:35.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6741221428571429, "accuracy_std_dev": 0.028652840424784446, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005719461038961039, "avg_duration_per_task_ms": 425768.6572871573, "avg_output_tokens_per_task": 15674.55266955267, "last_run_timestamp": "2026-10-10T16:20:00.451Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6725437394736843, "accuracy_std_dev": 0.2068414494859287, "run_count": 10, "total_questions": 1900, "avg_cost_per_task_usd": 0.013809886842105263, "avg_duration_per_task_ms": 353880.6878947368, "avg_output_tokens_per_task": 16644.034736842106, "last_run_timestamp": "2026-10-10T08:24:33.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6678543283847981, "accuracy_std_dev": 0.01614207710791971, "run_count": 9, "total_questions": 1684, "avg_cost_per_task_usd": 0.007296722090261282, "avg_duration_per_task_ms": 505935.9150831354, "avg_output_tokens_per_task": 51932.642517814726, "last_run_timestamp": "2026-10-10T16:25:08.075Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6678278397720075, "accuracy_std_dev": 0.09103865920988938, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.01146532425585814, "avg_duration_per_task_ms": 1011964.7701076631, "avg_output_tokens_per_task": 33559.86700443319, "last_run_timestamp": "2026-10-10T16:24:41.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.666857044520548, "accuracy_std_dev": 0.030362698141707424, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.0025397779680365294, "avg_duration_per_task_ms": 226314.88356164383, "avg_output_tokens_per_task": 5199.7414383561645, "last_run_timestamp": "2026-10-10T16:21:38.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009070929292929292, "avg_duration_per_task_ms": 492518.0909090909, "avg_output_tokens_per_task": 69555.08585858585, "last_run_timestamp": "2026-07-18T17:04:25.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.002231131313131313, "avg_duration_per_task_ms": 1386615.121212121, "avg_output_tokens_per_task": 14198.39898989899, "last_run_timestamp": "2026-07-18T09:58:41.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6666666666666666, "accuracy_std_dev": 0.1312433939450753, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.6057920415263749, "avg_duration_per_task_ms": 626284.2744107745, "avg_output_tokens_per_task": 40322.78170594837, "last_run_timestamp": "2026-10-10T08:23:04.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.6655442787878788, "accuracy_std_dev": 0.1380673949197909, "run_count": 8, "total_questions": 1485, "avg_cost_per_task_usd": 0.09031919797979798, "avg_duration_per_task_ms": 682014.4538720539, "avg_output_tokens_per_task": 28128.345454545455, "last_run_timestamp": "2026-10-10T16:11:14.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6632997142857142, "accuracy_std_dev": 0.009351528808768375, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0021804473304473304, "avg_duration_per_task_ms": 240867.02741702742, "avg_output_tokens_per_task": 3011.5793650793653, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.663019, "accuracy_std_dev": 0.007921062218330754, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0027949452861952863, "avg_duration_per_task_ms": 83787.81902356903, "avg_output_tokens_per_task": 2948.4612794612794, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6622475, "accuracy_std_dev": 0.023522840681983983, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02822281313131313, "avg_duration_per_task_ms": 831472.6881313132, "avg_output_tokens_per_task": 18950.592171717173, "last_run_timestamp": "2026-10-10T16:22:25.741Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6616161818181817, "accuracy_std_dev": 0.013544295262167041, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.005160062442607897, "avg_duration_per_task_ms": 106935.65518824609, "avg_output_tokens_per_task": 9720.085399449035, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6607744999999999, "accuracy_std_dev": 0.011572094721786573, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015565636363636365, "avg_duration_per_task_ms": 14159.460858585859, "avg_output_tokens_per_task": 1751.3977272727273, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.6595962, "accuracy_std_dev": 0.011187342962473291, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0024944898989898985, "avg_duration_per_task_ms": 89627.26666666666, "avg_output_tokens_per_task": 2311.8535353535353, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6579336250000001, "accuracy_std_dev": 0.01612810650260517, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.008984744633838386, "avg_duration_per_task_ms": 321798.5044191919, "avg_output_tokens_per_task": 59577.13005050505, "last_run_timestamp": "2026-10-10T16:21:42.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6564535333333333, "accuracy_std_dev": 0.13470588451282378, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.6553498678451178, "avg_duration_per_task_ms": 477953.10454545455, "avg_output_tokens_per_task": 40779.67929292929, "last_run_timestamp": "2026-10-10T08:22:15.381Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6558634490329324, "accuracy_std_dev": 0.13914333566507864, "run_count": 29, "total_questions": 5739, "avg_cost_per_task_usd": 0.6623900515769298, "avg_duration_per_task_ms": 598314.6978567694, "avg_output_tokens_per_task": 41294.878027530925, "last_run_timestamp": "2026-10-10T08:22:42.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.6548824, "accuracy_std_dev": 0.13698361654533725, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.6155487378787879, "avg_duration_per_task_ms": 631265.5565656566, "avg_output_tokens_per_task": 40882.59494949495, "last_run_timestamp": "2026-10-10T08:22:10.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.652862, "accuracy_std_dev": 0.023361257312054087, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007757086868686868, "avg_duration_per_task_ms": 821683.2373737374, "avg_output_tokens_per_task": 53689.36767676767, "last_run_timestamp": "2026-10-10T16:23:01.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.6515153, "accuracy_std_dev": 0.14316487590330945, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.6191192621212122, "avg_duration_per_task_ms": 634077.3176767677, "avg_output_tokens_per_task": 41194.81414141414, "last_run_timestamp": "2026-10-10T08:22:09.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6515151250000001, "accuracy_std_dev": 0.019686800377648354, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007399155303030303, "avg_duration_per_task_ms": 666816.1003787878, "avg_output_tokens_per_task": 57307.84595959596, "last_run_timestamp": "2026-10-10T16:27:33.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6512783352272727, "accuracy_std_dev": 0.07301920016707937, "run_count": 5, "total_questions": 704, "avg_cost_per_task_usd": 0.06126106249999999, "avg_duration_per_task_ms": 1570715.0482954546, "avg_output_tokens_per_task": 22881.873579545456, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.650363823580786, "accuracy_std_dev": 0.007103203700131033, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.0036315545851528385, "avg_duration_per_task_ms": 58503.03493449782, "avg_output_tokens_per_task": 2202.661135371179, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6495510000000001, "accuracy_std_dev": 0.00792134559276387, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0032256414141414147, "avg_duration_per_task_ms": 40985.67760942761, "avg_output_tokens_per_task": 3060.60101010101, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.649551, "accuracy_std_dev": 0.009444634067377443, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002492604377104377, "avg_duration_per_task_ms": 61525.246632996634, "avg_output_tokens_per_task": 2691.0799663299663, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.011195005920300148, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004075954545454546, "avg_duration_per_task_ms": 28668.510101010103, "avg_output_tokens_per_task": 2354.649831649832, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6473065000000001, "accuracy_std_dev": 0.009707729665065862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00413114393939394, "avg_duration_per_task_ms": 27377.72601010101, "avg_output_tokens_per_task": 2388.9305555555557, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6464646666666667, "accuracy_std_dev": 0.013537798302858865, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015368191919191919, "avg_duration_per_task_ms": 14060.520202020201, "avg_output_tokens_per_task": 1727.3535353535353, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.6461939736191261, "accuracy_std_dev": 0.08774312195800593, "run_count": 8, "total_questions": 1213, "avg_cost_per_task_usd": 0.046983095630667764, "avg_duration_per_task_ms": 297281.52844187967, "avg_output_tokens_per_task": 17813.622423742785, "last_run_timestamp": "2026-10-10T16:23:45.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6457914, "accuracy_std_dev": 0.017547304854022453, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007042358080808081, "avg_duration_per_task_ms": 238513.37575757576, "avg_output_tokens_per_task": 11322.903535353535, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6447479677419355, "accuracy_std_dev": 0.02407747186802543, "run_count": 3, "total_questions": 403, "avg_cost_per_task_usd": 0.003165789081885856, "avg_duration_per_task_ms": 173618.99503722086, "avg_output_tokens_per_task": 10324.468982630273, "last_run_timestamp": "2026-10-09T03:20:25.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6445006666666666, "accuracy_std_dev": 0.013998443861928202, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002421077441077441, "avg_duration_per_task_ms": 66682.89730639731, "avg_output_tokens_per_task": 2836.0850168350166, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.643659, "accuracy_std_dev": 0.015752669318774726, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015761657407407408, "avg_duration_per_task_ms": 18961.11531986532, "avg_output_tokens_per_task": 1683.8872053872053, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.643098, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018197469696969695, "avg_duration_per_task_ms": 1435371.2424242424, "avg_output_tokens_per_task": 32217.19191919192, "last_run_timestamp": "2026-07-18T17:46:02.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.6428570571428571, "accuracy_std_dev": 0.0682565, "run_count": 2, "total_questions": 175, "avg_cost_per_task_usd": 0.09669735428571428, "avg_duration_per_task_ms": 1267820.5257142857, "avg_output_tokens_per_task": 34805.99428571429, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6428055813449023, "accuracy_std_dev": 0.011146034629373067, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.0026296478669558935, "avg_duration_per_task_ms": 140713.84381778742, "avg_output_tokens_per_task": 9405.384671005062, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6413729024390243, "accuracy_std_dev": 0.03637250000000003, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.027402341463414633, "avg_duration_per_task_ms": 991300.8970189702, "avg_output_tokens_per_task": 32627.39024390244, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6406363579175706, "accuracy_std_dev": 0.016263383886158854, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.002550553145336226, "avg_duration_per_task_ms": 225461.96384671004, "avg_output_tokens_per_task": 8275.081706435285, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6405723333333334, "accuracy_std_dev": 0.0030349496792460334, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004210166666666667, "avg_duration_per_task_ms": 30431.28367003367, "avg_output_tokens_per_task": 2308.9638047138046, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.6402118571428571, "accuracy_std_dev": 0.01348951470511788, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004506121212121212, "avg_duration_per_task_ms": 72990.16883116883, "avg_output_tokens_per_task": 8472.02886002886, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.639316, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.0024375846153846152, "avg_duration_per_task_ms": 97724.68205128206, "avg_output_tokens_per_task": 7853.553846153846, "last_run_timestamp": "2026-07-18T08:33:31.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.638468, "accuracy_std_dev": 0.016031912152017304, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.009180828282828282, "avg_duration_per_task_ms": 658413.9090909091, "avg_output_tokens_per_task": 48820.804924242424, "last_run_timestamp": "2026-10-10T16:28:07.050Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6377313888888888, "accuracy_std_dev": 0.020028968176674062, "run_count": 6, "total_questions": 1152, "avg_cost_per_task_usd": 0.006115868055555556, "avg_duration_per_task_ms": 899818.5989583334, "avg_output_tokens_per_task": 35427.116319444445, "last_run_timestamp": "2026-10-10T08:14:41.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6371981347826087, "accuracy_std_dev": 0.014429669830892581, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.0032497253623188405, "avg_duration_per_task_ms": 218812.71086956523, "avg_output_tokens_per_task": 10563.355072463768, "last_run_timestamp": "2026-10-03T16:41:22.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6361772567678425, "accuracy_std_dev": 0.013327375155318935, "run_count": 7, "total_questions": 1219, "avg_cost_per_task_usd": 0.007504426579163248, "avg_duration_per_task_ms": 35368.206726825265, "avg_output_tokens_per_task": 12428.807219031993, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6356561135888502, "accuracy_std_dev": 0.014057249530380225, "run_count": 8, "total_questions": 1435, "avg_cost_per_task_usd": 0.01605337770034843, "avg_duration_per_task_ms": 91943.85017421603, "avg_output_tokens_per_task": 53289.633449477355, "last_run_timestamp": "2026-10-10T16:28:40.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6352257765567765, "accuracy_std_dev": 0.022072905759475025, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.013415796703296703, "avg_duration_per_task_ms": 548616.1373626373, "avg_output_tokens_per_task": 46655.3021978022, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6340491250000001, "accuracy_std_dev": 0.012397945832248788, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0016212228535353534, "avg_duration_per_task_ms": 88876.8813131313, "avg_output_tokens_per_task": 8195.30366161616, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.632856543563069, "accuracy_std_dev": 0.17446938590265632, "run_count": 4, "total_questions": 769, "avg_cost_per_task_usd": 0.06782110143042913, "avg_duration_per_task_ms": 1537627.533159948, "avg_output_tokens_per_task": 34138.841352405725, "last_run_timestamp": "2026-08-22T17:02:13.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6328437272727273, "accuracy_std_dev": 0.03335048624993999, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.006873024334251608, "avg_duration_per_task_ms": 112649.99265381084, "avg_output_tokens_per_task": 9586.507805325988, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6325426993295513, "accuracy_std_dev": 0.010534965676735725, "run_count": 10, "total_questions": 1939, "avg_cost_per_task_usd": 0.01766013512119649, "avg_duration_per_task_ms": 478451.96183599794, "avg_output_tokens_per_task": 88282.73388344508, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.631776382667417, "accuracy_std_dev": 0.016262402453375572, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.0017607208778840744, "avg_duration_per_task_ms": 208780.86043894204, "avg_output_tokens_per_task": 9645.604952166574, "last_run_timestamp": "2026-10-10T16:20:25.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.63118499609375, "accuracy_std_dev": 0.12107794090401254, "run_count": 3, "total_questions": 512, "avg_cost_per_task_usd": 0.00688423046875, "avg_duration_per_task_ms": 160958.9140625, "avg_output_tokens_per_task": 8535.359375, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6310326666666666, "accuracy_std_dev": 0.13772289129161577, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.615210808080808, "avg_duration_per_task_ms": 628271.4768518518, "avg_output_tokens_per_task": 40892.35058922559, "last_run_timestamp": "2026-10-10T08:24:11.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6305803258928572, "accuracy_std_dev": 0.21954975394425835, "run_count": 8, "total_questions": 1344, "avg_cost_per_task_usd": 0.0069273124999999994, "avg_duration_per_task_ms": 192378.60044642858, "avg_output_tokens_per_task": 11473.340773809523, "last_run_timestamp": "2026-10-10T16:21:31.775Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.6297015953307392, "accuracy_std_dev": 0.01592646788319292, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.023244770428015562, "avg_duration_per_task_ms": 1042943.9273670558, "avg_output_tokens_per_task": 96652.03437094683, "last_run_timestamp": "2026-10-03T17:27:37.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.62899825, "accuracy_std_dev": 0.007126819079189552, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.001126895202020202, "avg_duration_per_task_ms": 367383.1767676768, "avg_output_tokens_per_task": 11681.671717171717, "last_run_timestamp": "2026-10-10T16:22:34.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.628788, "accuracy_std_dev": 0.066498, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00551144696969697, "avg_duration_per_task_ms": 2569506.212121212, "avg_output_tokens_per_task": 42356.12373737374, "last_run_timestamp": "2026-10-10T16:26:33.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6285929916344916, "accuracy_std_dev": 0.03714754479584586, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.06302590733590734, "avg_duration_per_task_ms": 1075507.0637065638, "avg_output_tokens_per_task": 25670.111969111967, "last_run_timestamp": "2026-10-10T16:24:08.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6285795064570466, "accuracy_std_dev": 0.020919173358428876, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01718929421673217, "avg_duration_per_task_ms": 631165.4817518248, "avg_output_tokens_per_task": 83912.1010668164, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Mara", "accuracy": 0.628499, "accuracy_std_dev": null, "run_count": 1, "total_questions": 131, "avg_cost_per_task_usd": 0.0358695572519084, "avg_duration_per_task_ms": 72030.85496183205, "avg_output_tokens_per_task": 7685.93893129771, "last_run_timestamp": "2026-10-10T16:20:00.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6279425772773797, "accuracy_std_dev": 0.06414256831964248, "run_count": 5, "total_questions": 977, "avg_cost_per_task_usd": 0.012322606960081882, "avg_duration_per_task_ms": 1444518.9590583418, "avg_output_tokens_per_task": 53882.452405322416, "last_run_timestamp": "2026-10-10T16:20:00.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.626263, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012936333333333333, "avg_duration_per_task_ms": 385474.66161616164, "avg_output_tokens_per_task": 30893.848484848484, "last_run_timestamp": "2026-07-18T16:20:00.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.6261091711026616, "accuracy_std_dev": 0.05551308764606775, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.0414238669201521, "avg_duration_per_task_ms": 392853.54372623574, "avg_output_tokens_per_task": 20615.842839036755, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.623263890625, "accuracy_std_dev": 0.046540499999999985, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.03095198958333333, "avg_duration_per_task_ms": 834783.7890625, "avg_output_tokens_per_task": 38538.341145833336, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.623188452685422, "accuracy_std_dev": 0.0187657309831867, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.014858657289002557, "avg_duration_per_task_ms": 724190.4782608695, "avg_output_tokens_per_task": 15645.849104859335, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008206156565656566, "avg_duration_per_task_ms": 339475.95454545453, "avg_output_tokens_per_task": 15670.888888888889, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.6208752999999999, "accuracy_std_dev": 0.12687124759381063, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.002729046969696969, "avg_duration_per_task_ms": 232884.66262626264, "avg_output_tokens_per_task": 6365.909595959596, "last_run_timestamp": "2026-10-10T16:19:37.313Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "NextBit", "accuracy": 0.614478, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009234040404040404, "avg_duration_per_task_ms": 990972.4696969697, "avg_output_tokens_per_task": 16689.560606060608, "last_run_timestamp": "2026-07-18T08:03:56.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6128760749682338, "accuracy_std_dev": 0.007225322047320813, "run_count": 4, "total_questions": 787, "avg_cost_per_task_usd": 0.0033133367217280814, "avg_duration_per_task_ms": 404332.3011435832, "avg_output_tokens_per_task": 13398.312579415502, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6108306666666666, "accuracy_std_dev": 0.019131244624668038, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005598868686868688, "avg_duration_per_task_ms": 615283.2525252525, "avg_output_tokens_per_task": 18654.404882154882, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6102353048048047, "accuracy_std_dev": 0.004626267235105652, "run_count": 7, "total_questions": 1332, "avg_cost_per_task_usd": 0.004039552552552552, "avg_duration_per_task_ms": 36383.835585585584, "avg_output_tokens_per_task": 2146.3363363363364, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6096681428571429, "accuracy_std_dev": 0.010277069224089844, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0036427301587301586, "avg_duration_per_task_ms": 34466.44155844156, "avg_output_tokens_per_task": 2150.969696969697, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6096471504551988, "accuracy_std_dev": 0.016445311419549056, "run_count": 11, "total_questions": 2087, "avg_cost_per_task_usd": 0.008342372783900336, "avg_duration_per_task_ms": 542298.5462386201, "avg_output_tokens_per_task": 13701.597987541927, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6084389949367088, "accuracy_std_dev": 0.010283734794972733, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.008408201687763715, "avg_duration_per_task_ms": 279347.2447257384, "avg_output_tokens_per_task": 17442.14936708861, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6077443333333332, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0077788232323232325, "avg_duration_per_task_ms": 287812.98148148146, "avg_output_tokens_per_task": 15362.217171717171, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004114661616161616, "avg_duration_per_task_ms": 51474.35353535353, "avg_output_tokens_per_task": 2201.479797979798, "last_run_timestamp": "2026-07-18T08:26:39.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.6060605, "accuracy_std_dev": 0.001683499999999949, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008235411616161616, "avg_duration_per_task_ms": 152037.7323232323, "avg_output_tokens_per_task": 15653.954545454546, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.605239731707317, "accuracy_std_dev": 0.09170649999999997, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.03298509756097561, "avg_duration_per_task_ms": 869238.1111111111, "avg_output_tokens_per_task": 34362.55826558266, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6046178571428571, "accuracy_std_dev": 0.006995035043894045, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0034120865800865803, "avg_duration_per_task_ms": 53141.78787878788, "avg_output_tokens_per_task": 2114.74531024531, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6036557142857142, "accuracy_std_dev": 0.0048341465405498454, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011036557720057719, "avg_duration_per_task_ms": 33894.917748917745, "avg_output_tokens_per_task": 2102.8852813852814, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6026559059760956, "accuracy_std_dev": 0.018857781098506346, "run_count": 7, "total_questions": 1255, "avg_cost_per_task_usd": 0.002205658167330677, "avg_duration_per_task_ms": 47812.13147410358, "avg_output_tokens_per_task": 2066.2470119521913, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.6002888571428572, "accuracy_std_dev": 0.011234048798548233, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.003266399711399712, "avg_duration_per_task_ms": 60828.217893217894, "avg_output_tokens_per_task": 2084.8766233766232, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.599327, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014991068181818182, "avg_duration_per_task_ms": 284433.01515151514, "avg_output_tokens_per_task": 12302.075757575758, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.5980126829268292, "accuracy_std_dev": 0.061359, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.03206006233062331, "avg_duration_per_task_ms": 831219.6937669377, "avg_output_tokens_per_task": 40208.840108401084, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5973626666666666, "accuracy_std_dev": 0.014397638598819675, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003072445286195286, "avg_duration_per_task_ms": 655562.0867003367, "avg_output_tokens_per_task": 13595.005050505051, "last_run_timestamp": "2026-10-07T13:51:26.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.5968335954198474, "accuracy_std_dev": 0.034825528389147715, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.024714866836301953, "avg_duration_per_task_ms": 1074343.121289228, "avg_output_tokens_per_task": 64653.8074639525, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.591751, "accuracy_std_dev": 0.006627755628164546, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017204867003367003, "avg_duration_per_task_ms": 118615.4696969697, "avg_output_tokens_per_task": 42873.59259259259, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.5903911367346939, "accuracy_std_dev": 0.19755781409865822, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.024625327040816332, "avg_duration_per_task_ms": 733683.4362244898, "avg_output_tokens_per_task": 34746.054081632654, "last_run_timestamp": "2026-10-10T16:19:57.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5878479469525959, "accuracy_std_dev": 0.06702947204028986, "run_count": 10, "total_questions": 1772, "avg_cost_per_task_usd": 0.009048836343115122, "avg_duration_per_task_ms": 528814.486455982, "avg_output_tokens_per_task": 45050.80643340858, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.587542, "accuracy_std_dev": 0.205387, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0584044595959596, "avg_duration_per_task_ms": 1074676.5176767677, "avg_output_tokens_per_task": 23210.881313131315, "last_run_timestamp": "2026-10-10T16:11:14.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5860806989010988, "accuracy_std_dev": 0.014672256934467207, "run_count": 7, "total_questions": 1365, "avg_cost_per_task_usd": 0.006546478388278388, "avg_duration_per_task_ms": 728895.4901098901, "avg_output_tokens_per_task": 43384.46666666667, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5858587500000001, "accuracy_std_dev": 0.016059605768433433, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010333723484848484, "avg_duration_per_task_ms": 11960.345328282829, "avg_output_tokens_per_task": 13323.82702020202, "last_run_timestamp": "2026-10-10T16:19:40.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.582492, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010260318181818183, "avg_duration_per_task_ms": 317852.45454545453, "avg_output_tokens_per_task": 40768.580808080806, "last_run_timestamp": "2026-10-10T16:21:33.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.5786679990347491, "accuracy_std_dev": 0.10049636372121881, "run_count": 8, "total_questions": 1036, "avg_cost_per_task_usd": 0.0965155164092664, "avg_duration_per_task_ms": 1156721.8156370656, "avg_output_tokens_per_task": 27492.188223938225, "last_run_timestamp": "2026-10-10T16:27:46.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5750844, "accuracy_std_dev": 0.02978589572666902, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0254069303030303, "avg_duration_per_task_ms": 1275373.0656565656, "avg_output_tokens_per_task": 63407.47878787879, "last_run_timestamp": "2026-08-30T00:49:28.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5654762371428571, "accuracy_std_dev": 0.030454147358725023, "run_count": 7, "total_questions": 700, "avg_cost_per_task_usd": 0.032299544285714285, "avg_duration_per_task_ms": 59076.84, "avg_output_tokens_per_task": 6910.772857142857, "last_run_timestamp": "2026-10-10T16:21:24.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.564814935483871, "accuracy_std_dev": 0.015030477400564811, "run_count": 3, "total_questions": 558, "avg_cost_per_task_usd": 0.0051593978494623655, "avg_duration_per_task_ms": 615212.1326164874, "avg_output_tokens_per_task": 14268.974910394265, "last_run_timestamp": "2026-10-05T18:22:40.566Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.5633876206896551, "accuracy_std_dev": 0.09194318097738406, "run_count": 5, "total_questions": 986, "avg_cost_per_task_usd": 0.029487100405679513, "avg_duration_per_task_ms": 2640133.111561866, "avg_output_tokens_per_task": 23360.7738336714, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5630111428571428, "accuracy_std_dev": 0.01687958484110801, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004603796536796536, "avg_duration_per_task_ms": 419928.0497835498, "avg_output_tokens_per_task": 16214.744588744588, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5602958771331057, "accuracy_std_dev": 0.05661860675431709, "run_count": 5, "total_questions": 586, "avg_cost_per_task_usd": 0.05058588907849829, "avg_duration_per_task_ms": 611241.4914675768, "avg_output_tokens_per_task": 24217.037542662118, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.5560472579013906, "accuracy_std_dev": 0.1010106402129715, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.009295692793931732, "avg_duration_per_task_ms": 331772.0916561315, "avg_output_tokens_per_task": 27138.45195954488, "last_run_timestamp": "2026-10-10T16:20:03.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5505053333333333, "accuracy_std_dev": 0.016890890559378122, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.017397131313131312, "avg_duration_per_task_ms": 129348.1271043771, "avg_output_tokens_per_task": 43400.656565656565, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5488216666666668, "accuracy_std_dev": 0.006298902038362641, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0171891531986532, "avg_duration_per_task_ms": 121387.12121212122, "avg_output_tokens_per_task": 42892.78114478115, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.535985, "accuracy_std_dev": 0.011835000612589788, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0018682651515151514, "avg_duration_per_task_ms": 92043.92676767676, "avg_output_tokens_per_task": 1839.9911616161617, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.5353534999999999, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005200381313131313, "avg_duration_per_task_ms": 113966.90151515152, "avg_output_tokens_per_task": 11220.5, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.5331086666666667, "accuracy_std_dev": 0.05754091454106565, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005119237373737374, "avg_duration_per_task_ms": 101325.84511784512, "avg_output_tokens_per_task": 5420.193602693603, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5282142748691099, "accuracy_std_dev": 0.013706854874680634, "run_count": 6, "total_questions": 1146, "avg_cost_per_task_usd": 0.01651366666666667, "avg_duration_per_task_ms": 725814.7731239093, "avg_output_tokens_per_task": 41825.98516579407, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.522126, "accuracy_std_dev": 0.02474951208754282, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01514130735930736, "avg_duration_per_task_ms": 13491.472582972583, "avg_output_tokens_per_task": 1320.8239538239538, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.517469628909552, "accuracy_std_dev": 0.07510589055164418, "run_count": 6, "total_questions": 1183, "avg_cost_per_task_usd": 0.003801546914623838, "avg_duration_per_task_ms": 137539.94759087067, "avg_output_tokens_per_task": 4071.0912933220625, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.513468, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006161969696969697, "avg_duration_per_task_ms": 165410.29292929292, "avg_output_tokens_per_task": 12691.05303030303, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5080647741935485, "accuracy_std_dev": 0.12443414088451235, "run_count": 6, "total_questions": 1116, "avg_cost_per_task_usd": 0.004283293010752688, "avg_duration_per_task_ms": 107626.95967741935, "avg_output_tokens_per_task": 3879.1989247311826, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.5056116666666668, "accuracy_std_dev": 0.007570922855827376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012942734006734005, "avg_duration_per_task_ms": 12612.308080808081, "avg_output_tokens_per_task": 1100.3198653198654, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5, "accuracy_std_dev": 0.015120700182200542, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008362104377104377, "avg_duration_per_task_ms": 19874.548821548822, "avg_output_tokens_per_task": 1898.5521885521885, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.49764320000000006, "accuracy_std_dev": 0.01852187333290023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0008495191919191919, "avg_duration_per_task_ms": 13033.880808080808, "avg_output_tokens_per_task": 1851.6353535353535, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4966332500000001, "accuracy_std_dev": 0.014773079898501197, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008230315656565656, "avg_duration_per_task_ms": 17739.39898989899, "avg_output_tokens_per_task": 1863.909090909091, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.49601803668941985, "accuracy_std_dev": 0.017030085894772614, "run_count": 6, "total_questions": 1172, "avg_cost_per_task_usd": 0.01113635409556314, "avg_duration_per_task_ms": 463652.90699658706, "avg_output_tokens_per_task": 27776.748293515357, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.49551055555555557, "accuracy_std_dev": 0.015490728216948283, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.002358911335578002, "avg_duration_per_task_ms": 10696.016273849607, "avg_output_tokens_per_task": 2339.1021324354656, "last_run_timestamp": "2026-10-10T16:21:44.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.4939061636197441, "accuracy_std_dev": 0.024075339672231702, "run_count": 6, "total_questions": 1094, "avg_cost_per_task_usd": 0.010896170018281536, "avg_duration_per_task_ms": 477480.3711151737, "avg_output_tokens_per_task": 26830.819012797074, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.4855029999999999, "accuracy_std_dev": 0.0052309263679263035, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0023659646464646465, "avg_duration_per_task_ms": 14180.718294051627, "avg_output_tokens_per_task": 2226.1846240179575, "last_run_timestamp": "2026-10-10T16:21:33.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.48439725021570323, "accuracy_std_dev": 0.017142665996636056, "run_count": 12, "total_questions": 2318, "avg_cost_per_task_usd": 0.0011154288179465058, "avg_duration_per_task_ms": 60423.7131147541, "avg_output_tokens_per_task": 2138.6691113028473, "last_run_timestamp": "2026-10-10T16:22:21.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.48063949999999994, "accuracy_std_dev": 0.003857437653417109, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04852650126262626, "avg_duration_per_task_ms": 45991.458333333336, "avg_output_tokens_per_task": 17009.132575757576, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.4797977142857143, "accuracy_std_dev": 0.009264662382473756, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.002598592352092352, "avg_duration_per_task_ms": 54248.330447330445, "avg_output_tokens_per_task": 2851.073593073593, "last_run_timestamp": "2026-10-10T16:22:32.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4757993750000001, "accuracy_std_dev": 0.01160075218399113, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013241029040404041, "avg_duration_per_task_ms": 47333.95138888889, "avg_output_tokens_per_task": 2371.179924242424, "last_run_timestamp": "2026-10-10T16:25:26.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.4756825555555556, "accuracy_std_dev": 0.012142897971069449, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.003145421997755331, "avg_duration_per_task_ms": 58008.31144781145, "avg_output_tokens_per_task": 2218.3967452300785, "last_run_timestamp": "2026-10-10T16:27:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.4741861666666667, "accuracy_std_dev": 0.012808772416026279, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010600345117845118, "avg_duration_per_task_ms": 199985.06313131313, "avg_output_tokens_per_task": 2395.604377104377, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.47383958607594934, "accuracy_std_dev": 0.017544611925299883, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.0013290139240506329, "avg_duration_per_task_ms": 95497.43164556962, "avg_output_tokens_per_task": 3050.5791139240505, "last_run_timestamp": "2026-10-10T16:26:55.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.47264275, "accuracy_std_dev": 0.009698346982218158, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013737335858585856, "avg_duration_per_task_ms": 72305.53409090909, "avg_output_tokens_per_task": 2290.848484848485, "last_run_timestamp": "2026-10-10T16:23:00.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0022648939393939394, "avg_duration_per_task_ms": 33028.21717171717, "avg_output_tokens_per_task": 2380.9646464646466, "last_run_timestamp": "2026-07-18T16:13:31.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.46742483824768327, "accuracy_std_dev": 0.006948085005956674, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.002137747262005055, "avg_duration_per_task_ms": 34994.02274641955, "avg_output_tokens_per_task": 2587.217354675653, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46286933333333335, "accuracy_std_dev": 0.017560541816495547, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0007922811447811448, "avg_duration_per_task_ms": 158149.69472502806, "avg_output_tokens_per_task": 2225.2008978675644, "last_run_timestamp": "2026-10-10T16:20:22.051Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.45862900000000006, "accuracy_std_dev": null, "run_count": 1, "total_questions": 141, "avg_cost_per_task_usd": 0.0011888297872340425, "avg_duration_per_task_ms": 278701.0070921986, "avg_output_tokens_per_task": 7719.879432624113, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4320986666666667, "accuracy_std_dev": 0.00781631474687543, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010060690235690235, "avg_duration_per_task_ms": 20918.510101010103, "avg_output_tokens_per_task": 1483.5286195286196, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4315373333333334, "accuracy_std_dev": 0.010588221658469791, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010723358585858589, "avg_duration_per_task_ms": 21765.176767676767, "avg_output_tokens_per_task": 1505.858585858586, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4305555, "accuracy_std_dev": 0.007469952225416168, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010262436868686867, "avg_duration_per_task_ms": 17602.7702020202, "avg_output_tokens_per_task": 1517.9292929292928, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.40740725, "accuracy_std_dev": 0.3006428663691981, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017353968434343434, "avg_duration_per_task_ms": 189056.44444444444, "avg_output_tokens_per_task": 19318.219696969696, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.3994878450704225, "accuracy_std_dev": 0.0901063557941392, "run_count": 4, "total_questions": 781, "avg_cost_per_task_usd": 0.016465559539052498, "avg_duration_per_task_ms": 3369255.4609475033, "avg_output_tokens_per_task": 16587.451984635085, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.38720499999999997, "accuracy_std_dev": 0.010647388881786938, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00015740101010101008, "avg_duration_per_task_ms": 6591.242424242424, "avg_output_tokens_per_task": 937.0363636363636, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.37878799999999996, "accuracy_std_dev": 0.01683499999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000344469696969697, "avg_duration_per_task_ms": 46431.74494949495, "avg_output_tokens_per_task": 2036.358585858586, "last_run_timestamp": "2026-10-07T13:35:26.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.36698714615384614, "accuracy_std_dev": 0.09273293342139835, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.0036227448717948718, "avg_duration_per_task_ms": 36782.153846153844, "avg_output_tokens_per_task": 1520.774358974359, "last_run_timestamp": "2026-10-10T16:29:29.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.350168, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011657070707070707, "avg_duration_per_task_ms": 7587.606060606061, "avg_output_tokens_per_task": 355.9040404040404, "last_run_timestamp": "2026-07-18T09:09:45.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Novita", "accuracy": 0.344383, "accuracy_std_dev": null, "run_count": 1, "total_questions": 181, "avg_cost_per_task_usd": 0.00011365193370165745, "avg_duration_per_task_ms": 62171.872928176796, "avg_output_tokens_per_task": 485.27624309392263, "last_run_timestamp": "2026-07-18T08:00:15.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.338384, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00013078787878787878, "avg_duration_per_task_ms": 16627.883838383837, "avg_output_tokens_per_task": 667.8888888888889, "last_run_timestamp": "2026-07-18T09:08:52.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DeepInfra", "accuracy": 0.301347, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00002756060606060606, "avg_duration_per_task_ms": 15713.474747474747, "avg_output_tokens_per_task": 431.8787878787879, "last_run_timestamp": "2026-07-18T08:00:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.29292928571428567, "accuracy_std_dev": 0.07781645629981633, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008772483405483405, "avg_duration_per_task_ms": 593759.7330447331, "avg_output_tokens_per_task": 21699.420634920636, "last_run_timestamp": "2026-10-10T16:20:00.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DekaLLM", "accuracy": 0.289562, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00002257070707070707, "avg_duration_per_task_ms": 19325.131313131315, "avg_output_tokens_per_task": 285.6515151515151, "last_run_timestamp": "2026-07-18T08:04:03.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2891415, "accuracy_std_dev": 0.003232916137174003, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006112626262626263, "avg_duration_per_task_ms": 10017.040404040405, "avg_output_tokens_per_task": 577.7083333333334, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.284932625, "accuracy_std_dev": 0.018475559591914253, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0014604040404040406, "avg_duration_per_task_ms": 111179.26388888889, "avg_output_tokens_per_task": 19382.94191919192, "last_run_timestamp": "2026-10-10T16:10:32.137Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.279356125, "accuracy_std_dev": 0.011390538589521344, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0023889917929292926, "avg_duration_per_task_ms": 48269.878787878784, "avg_output_tokens_per_task": 29863.64267676768, "last_run_timestamp": "2026-10-10T16:10:51.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2739361560283688, "accuracy_std_dev": 0.008153796102974918, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.00006610283687943263, "avg_duration_per_task_ms": 8879.929078014184, "avg_output_tokens_per_task": 551.2074468085107, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.273148, "accuracy_std_dev": 0.006113657170630353, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006268181818181819, "avg_duration_per_task_ms": 9553.181818181818, "avg_output_tokens_per_task": 586.439393939394, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.27249684030418253, "accuracy_std_dev": 0.016430604143810795, "run_count": 7, "total_questions": 1315, "avg_cost_per_task_usd": 0.008003044866920151, "avg_duration_per_task_ms": 324114.34372623573, "avg_output_tokens_per_task": 35682.14980988593, "last_run_timestamp": "2026-10-10T16:10:51.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.2700681612244898, "accuracy_std_dev": 0.016752437809465226, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.00005762346938775511, "avg_duration_per_task_ms": 9734.870408163266, "avg_output_tokens_per_task": 576.8010204081633, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.26565141116751273, "accuracy_std_dev": 0.016394730800616743, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.00036760180485053583, "avg_duration_per_task_ms": 403603.9740552736, "avg_output_tokens_per_task": 8952.823463056966, "last_run_timestamp": "2026-10-10T16:11:11.296Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.25645355555555555, "accuracy_std_dev": 0.014994751949421879, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0004083338945005611, "avg_duration_per_task_ms": 68671.33221099888, "avg_output_tokens_per_task": 7843.052188552188, "last_run_timestamp": "2026-10-10T16:11:11.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.254209, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.001291050505050505, "avg_duration_per_task_ms": 40150.21212121212, "avg_output_tokens_per_task": 5062.732323232323, "last_run_timestamp": "2026-07-18T08:26:34.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.23148166666666667, "accuracy_std_dev": 0.10429433789947032, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0013719949494949493, "avg_duration_per_task_ms": 502712.0462962963, "avg_output_tokens_per_task": 4456.498316498317, "last_run_timestamp": "2026-08-29T17:16:14.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.1910775, "accuracy_std_dev": 0.11363615432122239, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0014224511784511782, "avg_duration_per_task_ms": 46218.05892255892, "avg_output_tokens_per_task": 1225.9587542087543, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.060606, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.000040161616161616165, "avg_duration_per_task_ms": 3671.1060606060605, "avg_output_tokens_per_task": 93.72222222222223, "last_run_timestamp": "2026-08-01T08:05:34.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.002245, "accuracy_std_dev": 0.0007933738084913065, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003512888888888889, "avg_duration_per_task_ms": 175891.9090909091, "avg_output_tokens_per_task": 3312.4141414141413, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034544065656565656, "avg_duration_per_task_ms": 164367.952020202, "avg_output_tokens_per_task": 3253.931818181818, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8368423578947368, "accuracy_std_dev": 0.014073500000000017, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.008221831578947368, "avg_duration_per_task_ms": 31565.694736842106, "avg_output_tokens_per_task": 9295.157894736842, "last_run_timestamp": "2026-10-10T11:02:59.339Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.826667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00673608, "avg_duration_per_task_ms": 350369.78, "avg_output_tokens_per_task": 14092.86, "last_run_timestamp": "2026-10-10T20:45:37.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.82, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.28041234, "avg_duration_per_task_ms": 27407.28, "avg_output_tokens_per_task": 12578.56, "last_run_timestamp": "2026-10-10T11:23:44.621Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8115940000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05019476086956522, "avg_duration_per_task_ms": 222941.58695652173, "avg_output_tokens_per_task": 15139.565217391304, "last_run_timestamp": "2026-10-10T19:11:12.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8053333, "accuracy_std_dev": 0.019043900162781787, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.8753633840000001, "avg_duration_per_task_ms": 132992.252, "avg_output_tokens_per_task": 5787.606, "last_run_timestamp": "2026-10-10T11:24:18.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8, "accuracy_std_dev": 0.02683269150495342, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.9213041, "avg_duration_per_task_ms": 126365.092, "avg_output_tokens_per_task": 5777.006, "last_run_timestamp": "2026-10-10T11:19:59.852Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.8, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.030123700000000003, "avg_duration_per_task_ms": 294712.06, "avg_output_tokens_per_task": 14851.14, "last_run_timestamp": "2026-07-18T08:03:40.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7991667499999999, "accuracy_std_dev": 0.021714744735720465, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.8790712225, "avg_duration_per_task_ms": 145295.465, "avg_output_tokens_per_task": 5765.4325, "last_run_timestamp": "2026-10-10T11:04:02.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7966665, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01733308, "avg_duration_per_task_ms": 494326.98, "avg_output_tokens_per_task": 23551.69, "last_run_timestamp": "2026-10-10T13:10:11.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.7955556666666667, "accuracy_std_dev": 0.00831485358994507, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05401102000000001, "avg_duration_per_task_ms": 879734.1466666666, "avg_output_tokens_per_task": 37622.22, "last_run_timestamp": "2026-10-10T13:25:18.383Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7953334, "accuracy_std_dev": 0.01739731013806443, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.811625548, "avg_duration_per_task_ms": 134771.192, "avg_output_tokens_per_task": 5750.948, "last_run_timestamp": "2026-10-10T11:20:49.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7921053684210526, "accuracy_std_dev": 0.021335845450199346, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.9452779715789473, "avg_duration_per_task_ms": 96650.52210526315, "avg_output_tokens_per_task": 5665.656842105263, "last_run_timestamp": "2026-10-10T11:36:10.174Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.7908167142857143, "accuracy_std_dev": 0.016180499999999987, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.030663948979591835, "avg_duration_per_task_ms": 84618.54081632652, "avg_output_tokens_per_task": 15463.275510204081, "last_run_timestamp": "2026-10-10T11:01:59.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.79, "accuracy_std_dev": 0.01666640000120001, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.016075525, "avg_duration_per_task_ms": 177598.75, "avg_output_tokens_per_task": 11878.23, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.7897090469798659, "accuracy_std_dev": 0.0266343317585071, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.24385301342281882, "avg_duration_per_task_ms": 51187.906040268455, "avg_output_tokens_per_task": 4637.738255033557, "last_run_timestamp": "2026-10-10T11:25:15.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7874807758887171, "accuracy_std_dev": 0.014215753702334737, "run_count": 13, "total_questions": 647, "avg_cost_per_task_usd": 0.10405211437403399, "avg_duration_per_task_ms": 139136.8485316847, "avg_output_tokens_per_task": 11436.242658423493, "last_run_timestamp": "2026-10-10T11:58:08.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7873333, "accuracy_std_dev": 0.02279383055148914, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.487091924, "avg_duration_per_task_ms": 126152.578, "avg_output_tokens_per_task": 7141.378, "last_run_timestamp": "2026-10-10T11:33:13.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.7871621216216216, "accuracy_std_dev": 0.015496260115123086, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06734989864864864, "avg_duration_per_task_ms": 463154.4662162162, "avg_output_tokens_per_task": 9087.885135135135, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14809964, "avg_duration_per_task_ms": 328566.04, "avg_output_tokens_per_task": 11598.26, "last_run_timestamp": "2026-07-18T08:04:04.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.7866669999999999, "accuracy_std_dev": 0.040000000000000036, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07300471, "avg_duration_per_task_ms": 912319.43, "avg_output_tokens_per_task": 13438.04, "last_run_timestamp": "2026-10-10T12:49:31.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7862320000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.029582717391304348, "avg_duration_per_task_ms": 2435445.304347826, "avg_output_tokens_per_task": 108721.58695652174, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7860360641891891, "accuracy_std_dev": 0.025554613476174253, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.09440423986486486, "avg_duration_per_task_ms": 381879.5878378378, "avg_output_tokens_per_task": 16484.91891891892, "last_run_timestamp": "2026-10-10T11:07:21.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7859532642140468, "accuracy_std_dev": 0.022060806735148097, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028781180602006688, "avg_duration_per_task_ms": 346700.6989966555, "avg_output_tokens_per_task": 11874.802675585284, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.785926, "accuracy_std_dev": 0.017900825710316006, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18531832888888886, "avg_duration_per_task_ms": 64060.64, "avg_output_tokens_per_task": 6337.593333333333, "last_run_timestamp": "2026-10-10T11:33:06.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7853334, "accuracy_std_dev": 0.008844443331267377, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08491233599999999, "avg_duration_per_task_ms": 41767.208, "avg_output_tokens_per_task": 4731.868, "last_run_timestamp": "2026-10-10T11:37:21.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7850339489795918, "accuracy_std_dev": 0.024279488005516096, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.08008161836734694, "avg_duration_per_task_ms": 1138745.7020408164, "avg_output_tokens_per_task": 26323.910204081632, "last_run_timestamp": "2026-10-10T19:24:40.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7847225, "accuracy_std_dev": 0.020833500000000005, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.11493318749999999, "avg_duration_per_task_ms": 147359.25, "avg_output_tokens_per_task": 14387.041666666666, "last_run_timestamp": "2026-10-01T16:00:58.391Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7844446666666666, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18011569333333333, "avg_duration_per_task_ms": 71496.32666666666, "avg_output_tokens_per_task": 6301.473333333333, "last_run_timestamp": "2026-10-10T11:00:15.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.7844443333333334, "accuracy_std_dev": 0.006285236475706792, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.15203402, "avg_duration_per_task_ms": 476027.4533333333, "avg_output_tokens_per_task": 12653, "last_run_timestamp": "2026-10-10T21:40:31.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.78416675, "accuracy_std_dev": 0.021327508971689602, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09952602499999999, "avg_duration_per_task_ms": 76992.08, "avg_output_tokens_per_task": 4752.155, "last_run_timestamp": "2026-10-10T11:49:24.601Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7837446052249636, "accuracy_std_dev": 0.019731934402518834, "run_count": 14, "total_questions": 689, "avg_cost_per_task_usd": 0.08364079680696661, "avg_duration_per_task_ms": 85376.37300435414, "avg_output_tokens_per_task": 4386.013062409289, "last_run_timestamp": "2026-10-10T11:59:03.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7837037777777778, "accuracy_std_dev": 0.02157059167672393, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06625726222222222, "avg_duration_per_task_ms": 303688.90222222224, "avg_output_tokens_per_task": 15392.597777777777, "last_run_timestamp": "2026-10-10T20:56:17.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7827380586734692, "accuracy_std_dev": 0.0387434346968538, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.11838895663265307, "avg_duration_per_task_ms": 3017033.7933673467, "avg_output_tokens_per_task": 26124.95918367347, "last_run_timestamp": "2026-10-10T19:28:36.252Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7827005274261605, "accuracy_std_dev": 0.02374291531720566, "run_count": 5, "total_questions": 237, "avg_cost_per_task_usd": 0.10862543037974683, "avg_duration_per_task_ms": 71286.46835443038, "avg_output_tokens_per_task": 8041.662447257384, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.782500125, "accuracy_std_dev": 0.028662343424943032, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.19210731, "avg_duration_per_task_ms": 737716.5375, "avg_output_tokens_per_task": 31714.73, "last_run_timestamp": "2026-10-10T18:35:22.054Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7822223333333334, "accuracy_std_dev": 0.021998563367234276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011777086666666667, "avg_duration_per_task_ms": 550585.4066666666, "avg_output_tokens_per_task": 22732.366666666665, "last_run_timestamp": "2026-10-10T13:14:05.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.7822221666666667, "accuracy_std_dev": 0.012570738116974498, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.1844831233333333, "avg_duration_per_task_ms": 71285.40333333334, "avg_output_tokens_per_task": 6291.4, "last_run_timestamp": "2026-10-10T11:08:41.713Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.7822221666666667, "accuracy_std_dev": 0.017915990515148437, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07431559666666666, "avg_duration_per_task_ms": 47439.64666666667, "avg_output_tokens_per_task": 4714.83, "last_run_timestamp": "2026-10-10T11:39:52.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.7822220000000001, "accuracy_std_dev": 0.020608136952831674, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0806628, "avg_duration_per_task_ms": 57888.54, "avg_output_tokens_per_task": 12516.78, "last_run_timestamp": "2026-10-10T11:19:52.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.78166675, "accuracy_std_dev": 0.022985462799071505, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.47495438199999995, "avg_duration_per_task_ms": 107875.898, "avg_output_tokens_per_task": 7230.244, "last_run_timestamp": "2026-10-10T11:20:52.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7816666250000001, "accuracy_std_dev": 0.02303379461756953, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.48336481500000006, "avg_duration_per_task_ms": 127661.045, "avg_output_tokens_per_task": 7945.53, "last_run_timestamp": "2026-10-10T10:58:55.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.7813332000000001, "accuracy_std_dev": 0.03357245431838428, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.03929822, "avg_duration_per_task_ms": 339374.972, "avg_output_tokens_per_task": 12667.496, "last_run_timestamp": "2026-10-10T11:49:13.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.7811448181818181, "accuracy_std_dev": 0.0011565000000000047, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.035013020202020205, "avg_duration_per_task_ms": 141478.38383838383, "avg_output_tokens_per_task": 23821.878787878788, "last_run_timestamp": "2026-10-10T11:21:33.807Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7803333, "accuracy_std_dev": 0.024333256165379916, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.13113940600000001, "avg_duration_per_task_ms": 299273.458, "avg_output_tokens_per_task": 19639.796, "last_run_timestamp": "2026-10-10T12:00:37.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7800003333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2757919799999997, "avg_duration_per_task_ms": 102078.39333333333, "avg_output_tokens_per_task": 5685.653333333334, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.78, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.15629982, "avg_duration_per_task_ms": 82269.59, "avg_output_tokens_per_task": 6332.78, "last_run_timestamp": "2026-10-10T11:25:14.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.78, "accuracy_std_dev": 0.029514395734962978, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.086451392, "avg_duration_per_task_ms": 268938.36, "avg_output_tokens_per_task": 20045.732, "last_run_timestamp": "2026-10-10T11:31:04.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.78, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04487794, "avg_duration_per_task_ms": 324016.02, "avg_output_tokens_per_task": 12761.91, "last_run_timestamp": "2026-10-10T11:24:10.824Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12014696000000001, "avg_duration_per_task_ms": 363719.42, "avg_output_tokens_per_task": 9566.28, "last_run_timestamp": "2026-07-18T16:00:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7796420402684564, "accuracy_std_dev": 0.01821392930104136, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.15591147986577178, "avg_duration_per_task_ms": 329823.3187919463, "avg_output_tokens_per_task": 20298.120805369126, "last_run_timestamp": "2026-10-10T11:36:39.823Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7782795483870968, "accuracy_std_dev": 0.024653522574764702, "run_count": 31, "total_questions": 1550, "avg_cost_per_task_usd": 0.5512626245161292, "avg_duration_per_task_ms": 114385.19612903227, "avg_output_tokens_per_task": 7228.9838709677415, "last_run_timestamp": "2026-10-10T11:19:53.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7780952857142857, "accuracy_std_dev": 0.025125589310582973, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.15667032, "avg_duration_per_task_ms": 579481.5971428572, "avg_output_tokens_per_task": 12471.537142857143, "last_run_timestamp": "2026-10-10T12:12:40.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7780951428571429, "accuracy_std_dev": 0.020381833538918, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08139844285714284, "avg_duration_per_task_ms": 136861.11714285714, "avg_output_tokens_per_task": 4631.26, "last_run_timestamp": "2026-10-10T12:31:22.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7780950000000001, "accuracy_std_dev": 0.016605340233964306, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09330751714285715, "avg_duration_per_task_ms": 138037.12, "avg_output_tokens_per_task": 4580.668571428571, "last_run_timestamp": "2026-10-10T11:56:05.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.778, "accuracy_std_dev": 0.021578397978842947, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.48919124333333336, "avg_duration_per_task_ms": 124186.09133333333, "avg_output_tokens_per_task": 7191.306666666666, "last_run_timestamp": "2026-10-10T11:23:03.579Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.777037, "accuracy_std_dev": 0.017529048595593164, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12491927333333332, "avg_duration_per_task_ms": 86437.00444444445, "avg_output_tokens_per_task": 4516.131111111111, "last_run_timestamp": "2026-10-10T11:52:54.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7770368888888889, "accuracy_std_dev": 0.04333181447195696, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11806641555555555, "avg_duration_per_task_ms": 199701.67777777778, "avg_output_tokens_per_task": 7078.295555555555, "last_run_timestamp": "2026-10-10T11:52:41.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7769698181818182, "accuracy_std_dev": 0.020570475514096246, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4460740036363637, "avg_duration_per_task_ms": 130580.85818181818, "avg_output_tokens_per_task": 7285.6563636363635, "last_run_timestamp": "2026-10-10T11:20:01.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7769230769230769, "accuracy_std_dev": 0.0234780981226253, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.4968872015384615, "avg_duration_per_task_ms": 128094.2323076923, "avg_output_tokens_per_task": 8089.72, "last_run_timestamp": "2026-10-10T11:29:58.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7761905714285714, "accuracy_std_dev": 0.025161623589773956, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07583003714285713, "avg_duration_per_task_ms": 286315.0457142857, "avg_output_tokens_per_task": 13935.274285714286, "last_run_timestamp": "2026-10-10T11:31:34.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.776, "accuracy_std_dev": 0.02004423996064706, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.114664068, "avg_duration_per_task_ms": 365492.12, "avg_output_tokens_per_task": 20755.204, "last_run_timestamp": "2026-10-10T12:02:52.502Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7759258888888889, "accuracy_std_dev": 0.02243107168116209, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18105771111111107, "avg_duration_per_task_ms": 381492.62, "avg_output_tokens_per_task": 28609.035555555554, "last_run_timestamp": "2026-10-10T18:21:48.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7755556666666666, "accuracy_std_dev": 0.027932599656236, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.17062818666666668, "avg_duration_per_task_ms": 84211.09333333334, "avg_output_tokens_per_task": 6269.78, "last_run_timestamp": "2026-10-10T11:20:55.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7752381428571429, "accuracy_std_dev": 0.01841823507465649, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06935552285714286, "avg_duration_per_task_ms": 37222.497142857144, "avg_output_tokens_per_task": 4756.58, "last_run_timestamp": "2026-10-10T11:29:01.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7750876842105263, "accuracy_std_dev": 0.028681883484458735, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.5084564494736843, "avg_duration_per_task_ms": 128153.6252631579, "avg_output_tokens_per_task": 7952.632631578947, "last_run_timestamp": "2026-10-10T11:20:59.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7749997500000001, "accuracy_std_dev": 0.014433785597946898, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.056739235, "avg_duration_per_task_ms": 137409.03, "avg_output_tokens_per_task": 4567.525, "last_run_timestamp": "2026-10-10T11:51:41.085Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.7748147777777777, "accuracy_std_dev": 0.0399725818246507, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11767610666666668, "avg_duration_per_task_ms": 282128.4533333333, "avg_output_tokens_per_task": 12261.444444444445, "last_run_timestamp": "2026-10-10T22:52:53.325Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.7748147777777777, "accuracy_std_dev": 0.029275825597155438, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02169746888888889, "avg_duration_per_task_ms": 300435.5488888889, "avg_output_tokens_per_task": 11767.76, "last_run_timestamp": "2026-10-10T19:46:14.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7747070301507537, "accuracy_std_dev": 0.01996857802899347, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.06471272864321607, "avg_duration_per_task_ms": 1090207.1457286433, "avg_output_tokens_per_task": 32780.115577889446, "last_run_timestamp": "2026-10-10T11:21:41.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7746668, "accuracy_std_dev": 0.0165463013679795, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.153314064, "avg_duration_per_task_ms": 285965.152, "avg_output_tokens_per_task": 19906.316, "last_run_timestamp": "2026-10-10T11:39:25.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.7744443333333334, "accuracy_std_dev": 0.019404651346749013, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08295856333333333, "avg_duration_per_task_ms": 90230.37333333334, "avg_output_tokens_per_task": 4538.593333333333, "last_run_timestamp": "2026-10-10T11:21:34.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7743589230769231, "accuracy_std_dev": 0.017314924986894253, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.4353327676923078, "avg_duration_per_task_ms": 95008.13230769231, "avg_output_tokens_per_task": 5676.784615384616, "last_run_timestamp": "2026-10-10T11:19:17.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.026945723560596418, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21623159142857146, "avg_duration_per_task_ms": 562306.5571428571, "avg_output_tokens_per_task": 12602.945714285714, "last_run_timestamp": "2026-10-10T11:06:43.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.774166625, "accuracy_std_dev": 0.017460369812360068, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10507661, "avg_duration_per_task_ms": 319305.445, "avg_output_tokens_per_task": 20731.5575, "last_run_timestamp": "2026-10-10T11:20:55.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.774, "accuracy_std_dev": 0.026574933888911183, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04397801800000001, "avg_duration_per_task_ms": 214737.652, "avg_output_tokens_per_task": 9570.176, "last_run_timestamp": "2026-10-10T19:06:48.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.77399995, "accuracy_std_dev": 0.031048382675229646, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.444216434, "avg_duration_per_task_ms": 79552.85, "avg_output_tokens_per_task": 5673.027, "last_run_timestamp": "2026-10-10T11:48:23.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.7733334, "accuracy_std_dev": 0.01837863493951607, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09612659999999999, "avg_duration_per_task_ms": 122579.768, "avg_output_tokens_per_task": 5303.732, "last_run_timestamp": "2026-10-10T11:43:14.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7733333, "accuracy_std_dev": 0.027648644733693543, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.4732467780000001, "avg_duration_per_task_ms": 113874.772, "avg_output_tokens_per_task": 7213.052, "last_run_timestamp": "2026-10-10T11:27:45.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.77333325, "accuracy_std_dev": 0.004713927363409425, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007096445, "avg_duration_per_task_ms": 164831.185, "avg_output_tokens_per_task": 17141.055, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11157742000000001, "avg_duration_per_task_ms": 52237.66, "avg_output_tokens_per_task": 12874.88, "last_run_timestamp": "2026-07-18T08:03:41.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02012318, "avg_duration_per_task_ms": 229027.46, "avg_output_tokens_per_task": 10940.04, "last_run_timestamp": "2026-07-18T08:04:46.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.042562220000000005, "avg_duration_per_task_ms": 321425.02, "avg_output_tokens_per_task": 8985.7, "last_run_timestamp": "2026-07-18T16:00:10.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7732656294416244, "accuracy_std_dev": 0.03435466996639615, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.010553837563451776, "avg_duration_per_task_ms": 1118057.5126903553, "avg_output_tokens_per_task": 17178.984771573603, "last_run_timestamp": "2026-10-10T11:52:41.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.772857, "accuracy_std_dev": 0.03077969484207776, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11466776285714285, "avg_duration_per_task_ms": 170975.38857142857, "avg_output_tokens_per_task": 9449.94, "last_run_timestamp": "2026-10-10T19:58:43.743Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.7725, "accuracy_std_dev": 0.019561299662854723, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17478934, "avg_duration_per_task_ms": 739323.4375, "avg_output_tokens_per_task": 28891.6325, "last_run_timestamp": "2026-10-10T18:38:21.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7722223333333333, "accuracy_std_dev": 0.010482248369929469, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11473854333333333, "avg_duration_per_task_ms": 271731.52, "avg_output_tokens_per_task": 23286.896666666667, "last_run_timestamp": "2026-10-10T11:07:22.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7722222222222223, "accuracy_std_dev": 0.03095202181433487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03593436666666667, "avg_duration_per_task_ms": 446735.1666666667, "avg_output_tokens_per_task": 13815.786666666667, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.772109, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.06601828571428571, "avg_duration_per_task_ms": 84701.85714285714, "avg_output_tokens_per_task": 5801.673469387755, "last_run_timestamp": "2026-07-18T08:06:26.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.772, "accuracy_std_dev": 0.010666750001757801, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.029716368000000003, "avg_duration_per_task_ms": 111311.872, "avg_output_tokens_per_task": 12081.752, "last_run_timestamp": "2026-10-10T18:37:53.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7719999000000001, "accuracy_std_dev": 0.019955421431029706, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.43238272600000005, "avg_duration_per_task_ms": 99679.176, "avg_output_tokens_per_task": 5611.424, "last_run_timestamp": "2026-10-10T11:06:20.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.771905, "accuracy_std_dev": 0.035315195576004266, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1764812885714286, "avg_duration_per_task_ms": 513263.16857142857, "avg_output_tokens_per_task": 25304.974285714285, "last_run_timestamp": "2026-10-10T18:50:47.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7717949230769231, "accuracy_std_dev": 0.0290184402604263, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.5512982523076922, "avg_duration_per_task_ms": 183923.44307692308, "avg_output_tokens_per_task": 10555.14, "last_run_timestamp": "2026-10-10T11:29:26.517Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7711111111111111, "accuracy_std_dev": 0.02828443711240216, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18611659777777775, "avg_duration_per_task_ms": 180574.86666666667, "avg_output_tokens_per_task": 9565.442222222222, "last_run_timestamp": "2026-10-10T18:32:40.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.771111, "accuracy_std_dev": 0.04085521560339636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4149127733333333, "avg_duration_per_task_ms": 472514.2266666667, "avg_output_tokens_per_task": 36470.113333333335, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.77083325, "accuracy_std_dev": 0.021327567581360528, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08374891499999998, "avg_duration_per_task_ms": 335021.265, "avg_output_tokens_per_task": 20027.295, "last_run_timestamp": "2026-10-10T12:20:33.626Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.770833, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.07522506250000001, "avg_duration_per_task_ms": 171662.70833333334, "avg_output_tokens_per_task": 11489.854166666666, "last_run_timestamp": "2026-10-10T18:58:05.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7704761428571428, "accuracy_std_dev": 0.021338487899491252, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008933357142857144, "avg_duration_per_task_ms": 211052.8457142857, "avg_output_tokens_per_task": 4263.237142857143, "last_run_timestamp": "2026-10-10T12:13:04.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.770476, "accuracy_std_dev": 0.02332836876300735, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07879788857142857, "avg_duration_per_task_ms": 228548.80857142858, "avg_output_tokens_per_task": 9318.671428571428, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7704113333333333, "accuracy_std_dev": 0.022998343056737182, "run_count": 11, "total_questions": 543, "avg_cost_per_task_usd": 0.05714349907918969, "avg_duration_per_task_ms": 253713.67034990792, "avg_output_tokens_per_task": 11037.460405156538, "last_run_timestamp": "2026-10-10T11:10:08.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7700001249999999, "accuracy_std_dev": 0.03395262221993134, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.071411165, "avg_duration_per_task_ms": 255968.725, "avg_output_tokens_per_task": 11897.3, "last_run_timestamp": "2026-10-10T11:38:42.065Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7700001, "accuracy_std_dev": 0.020923293772491944, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5362019739999999, "avg_duration_per_task_ms": 175302.906, "avg_output_tokens_per_task": 10636.738, "last_run_timestamp": "2026-10-10T11:19:29.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.77, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08223393, "avg_duration_per_task_ms": 429468.14, "avg_output_tokens_per_task": 14260.94, "last_run_timestamp": "2026-10-10T11:20:22.675Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7696296666666667, "accuracy_std_dev": 0.017245185808090188, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04810054666666667, "avg_duration_per_task_ms": 199896.38444444444, "avg_output_tokens_per_task": 13265.613333333333, "last_run_timestamp": "2026-10-10T19:37:47.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.7696296666666667, "accuracy_std_dev": 0.030161915139607597, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.011042515555555554, "avg_duration_per_task_ms": 262078.49333333335, "avg_output_tokens_per_task": 6314.64, "last_run_timestamp": "2026-10-10T11:50:30.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7695238571428571, "accuracy_std_dev": 0.03282612614727386, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08224388857142857, "avg_duration_per_task_ms": 49807.68, "avg_output_tokens_per_task": 4863.0028571428575, "last_run_timestamp": "2026-10-10T11:40:28.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7694871538461538, "accuracy_std_dev": 0.0298099088245868, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.052062487692307695, "avg_duration_per_task_ms": 95666.30307692308, "avg_output_tokens_per_task": 4338.936923076923, "last_run_timestamp": "2026-10-10T11:52:30.774Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7693334, "accuracy_std_dev": 0.016110772105644107, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.15629164, "avg_duration_per_task_ms": 313958.944, "avg_output_tokens_per_task": 15113.092, "last_run_timestamp": "2026-10-10T21:14:14.916Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.7693334, "accuracy_std_dev": 0.027194796889110983, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.047570167999999996, "avg_duration_per_task_ms": 114696.804, "avg_output_tokens_per_task": 4956.572, "last_run_timestamp": "2026-10-08T23:21:59.850Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7693333333333333, "accuracy_std_dev": 0.019444111745090222, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.06786480933333333, "avg_duration_per_task_ms": 55615.890666666666, "avg_output_tokens_per_task": 4658.706666666667, "last_run_timestamp": "2026-10-10T11:39:51.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7689700975609757, "accuracy_std_dev": 0.020642326587863087, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.06932281300813009, "avg_duration_per_task_ms": 206430.17479674798, "avg_output_tokens_per_task": 10777.69105691057, "last_run_timestamp": "2026-10-10T19:04:05.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7688889999999999, "accuracy_std_dev": 0.029979235614004583, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028244313333333337, "avg_duration_per_task_ms": 171006.77333333335, "avg_output_tokens_per_task": 15371.96, "last_run_timestamp": "2026-10-10T11:30:53.804Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7688889999999999, "accuracy_std_dev": 0.026574014970769236, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06194356999999999, "avg_duration_per_task_ms": 131938.69666666666, "avg_output_tokens_per_task": 6466.09, "last_run_timestamp": "2026-10-10T11:43:14.705Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7688888888888888, "accuracy_std_dev": 0.02703454580283221, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09845500888888889, "avg_duration_per_task_ms": 231449.82222222222, "avg_output_tokens_per_task": 11662.893333333333, "last_run_timestamp": "2026-10-10T18:58:28.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7688888333333334, "accuracy_std_dev": 0.017916093876890582, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.060097880000000006, "avg_duration_per_task_ms": 48342.426666666666, "avg_output_tokens_per_task": 4654.916666666667, "last_run_timestamp": "2026-10-10T11:20:35.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025211093333333334, "avg_duration_per_task_ms": 135029.63333333333, "avg_output_tokens_per_task": 13854.966666666667, "last_run_timestamp": "2026-10-10T18:35:38.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.021998731726069008, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09783254, "avg_duration_per_task_ms": 102292.53333333334, "avg_output_tokens_per_task": 23933.02, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.768678143678161, "accuracy_std_dev": 0.0271195517707225, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.03471193247126437, "avg_duration_per_task_ms": 522555.5474137931, "avg_output_tokens_per_task": 12696.757183908046, "last_run_timestamp": "2026-10-10T19:16:07.502Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7685808243243244, "accuracy_std_dev": 0.028647198085308245, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.07983985472972972, "avg_duration_per_task_ms": 256413, "avg_output_tokens_per_task": 10008.462837837838, "last_run_timestamp": "2026-10-10T19:17:08.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7683335, "accuracy_std_dev": 0.014433698997485036, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03325037, "avg_duration_per_task_ms": 273019.26, "avg_output_tokens_per_task": 39890.13, "last_run_timestamp": "2026-10-10T11:29:19.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7683109103139014, "accuracy_std_dev": 0.04045368552087958, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.027139372197309417, "avg_duration_per_task_ms": 484571.0381165919, "avg_output_tokens_per_task": 12770.852017937219, "last_run_timestamp": "2026-10-10T19:09:44.232Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7680952857142856, "accuracy_std_dev": 0.02110869827626438, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.017966814285714287, "avg_duration_per_task_ms": 193611, "avg_output_tokens_per_task": 19085.56142857143, "last_run_timestamp": "2026-10-10T11:05:58.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7680458965517241, "accuracy_std_dev": 0.020370441569782412, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.5334861268965518, "avg_duration_per_task_ms": 121001.88, "avg_output_tokens_per_task": 8118.9544827586205, "last_run_timestamp": "2026-10-10T11:33:22.028Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.768, "accuracy_std_dev": 0.011469922022402766, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09600952000000001, "avg_duration_per_task_ms": 63394.68, "avg_output_tokens_per_task": 4831.208, "last_run_timestamp": "2026-10-10T12:14:36.151Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7678787272727273, "accuracy_std_dev": 0.02555520860657607, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08920051818181818, "avg_duration_per_task_ms": 190464.52, "avg_output_tokens_per_task": 11544.347272727273, "last_run_timestamp": "2026-10-10T20:24:33.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.767778, "accuracy_std_dev": 0.009557997105391226, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.055267166666666666, "avg_duration_per_task_ms": 921928.7333333333, "avg_output_tokens_per_task": 38360.306666666664, "last_run_timestamp": "2026-10-03T08:15:16.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7677778333333333, "accuracy_std_dev": 0.01822341121027811, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.058372879999999995, "avg_duration_per_task_ms": 136563.96666666667, "avg_output_tokens_per_task": 14796.806666666667, "last_run_timestamp": "2026-10-10T11:05:27.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7675, "accuracy_std_dev": 0.0239067401008586, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.026842017500000006, "avg_duration_per_task_ms": 165041.6625, "avg_output_tokens_per_task": 8909.3125, "last_run_timestamp": "2026-10-10T18:58:24.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7673335, "accuracy_std_dev": 0.020965394769715172, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.49187341399999995, "avg_duration_per_task_ms": 131219.752, "avg_output_tokens_per_task": 8161.99, "last_run_timestamp": "2026-10-10T11:05:25.784Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025223939999999997, "avg_duration_per_task_ms": 165137.1, "avg_output_tokens_per_task": 14240.82, "last_run_timestamp": "2026-09-15T00:00:05.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0668136, "avg_duration_per_task_ms": 100184.26, "avg_output_tokens_per_task": 11544.38, "last_run_timestamp": "2026-07-18T08:04:03.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7666667777777777, "accuracy_std_dev": 0.028458217152628754, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.13082103333333334, "avg_duration_per_task_ms": 266780, "avg_output_tokens_per_task": 11720.468888888889, "last_run_timestamp": "2026-10-10T18:44:45.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7666667142857143, "accuracy_std_dev": 0.02108204682875446, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.058299279999999995, "avg_duration_per_task_ms": 114981.15428571429, "avg_output_tokens_per_task": 4748.437142857143, "last_run_timestamp": "2026-10-10T11:50:44.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18197839999999998, "avg_duration_per_task_ms": 75193.68666666666, "avg_output_tokens_per_task": 6381.386666666666, "last_run_timestamp": "2026-10-10T11:01:03.350Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.014401440259764175, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3460653, "avg_duration_per_task_ms": 170899.55333333334, "avg_output_tokens_per_task": 12611.153333333334, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.766666625, "accuracy_std_dev": 0.03480106160714606, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.434086625, "avg_duration_per_task_ms": 94242.2325, "avg_output_tokens_per_task": 5557.605, "last_run_timestamp": "2026-10-10T11:08:25.975Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.766666625, "accuracy_std_dev": 0.036055535868079636, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07008643250000002, "avg_duration_per_task_ms": 305737.3925, "avg_output_tokens_per_task": 12550.4925, "last_run_timestamp": "2026-10-10T11:39:51.190Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7666666, "accuracy_std_dev": 0.03319954484386795, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.032209988, "avg_duration_per_task_ms": 207977.044, "avg_output_tokens_per_task": 14953.416, "last_run_timestamp": "2026-10-10T11:06:17.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7666666, "accuracy_std_dev": 0.022705633174170663, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.013304756000000001, "avg_duration_per_task_ms": 229500.18, "avg_output_tokens_per_task": 4888.664, "last_run_timestamp": "2026-10-10T12:10:15.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7666665714285714, "accuracy_std_dev": 0.01851637627967774, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.019599598571428567, "avg_duration_per_task_ms": 240323.45, "avg_output_tokens_per_task": 16987.242857142857, "last_run_timestamp": "2026-10-10T12:00:58.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.012833540000000001, "avg_duration_per_task_ms": 88757.68, "avg_output_tokens_per_task": 11232.71, "last_run_timestamp": "2026-10-10T11:24:21.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01488279, "avg_duration_per_task_ms": 714098.21, "avg_output_tokens_per_task": 29412.46, "last_run_timestamp": "2026-10-10T13:27:56.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.7666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01352679, "avg_duration_per_task_ms": 90848.6, "avg_output_tokens_per_task": 4507.38, "last_run_timestamp": "2026-10-10T11:18:50.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7664834065934065, "accuracy_std_dev": 0.03806138445886618, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.10488499816849818, "avg_duration_per_task_ms": 804500.0018315018, "avg_output_tokens_per_task": 10329.6336996337, "last_run_timestamp": "2026-10-10T19:42:22.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.76633335, "accuracy_std_dev": 0.03073014101379133, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.567469223, "avg_duration_per_task_ms": 181854.152, "avg_output_tokens_per_task": 10594.497, "last_run_timestamp": "2026-10-10T11:40:33.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7662836379310344, "accuracy_std_dev": 0.02598916979066786, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.10895231896551724, "avg_duration_per_task_ms": 46529.12356321839, "avg_output_tokens_per_task": 4239.747126436781, "last_run_timestamp": "2026-10-10T11:35:23.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7661904285714286, "accuracy_std_dev": 0.01872337959311179, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019808859999999998, "avg_duration_per_task_ms": 510174.85142857145, "avg_output_tokens_per_task": 13353.934285714286, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7655556666666666, "accuracy_std_dev": 0.020518134106741345, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05405915333333334, "avg_duration_per_task_ms": 120906.59666666666, "avg_output_tokens_per_task": 4450.843333333333, "last_run_timestamp": "2026-10-10T11:50:02.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7655172758620691, "accuracy_std_dev": 0.02543532653057561, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.5846374510344827, "avg_duration_per_task_ms": 118911.25586206897, "avg_output_tokens_per_task": 8140.452413793103, "last_run_timestamp": "2026-10-10T11:25:12.451Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7654544545454546, "accuracy_std_dev": 0.028258289861283148, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.16681981636363638, "avg_duration_per_task_ms": 582078.4036363637, "avg_output_tokens_per_task": 36559.66, "last_run_timestamp": "2026-10-10T11:50:56.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7653332, "accuracy_std_dev": 0.02646590427247858, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019831184, "avg_duration_per_task_ms": 375594.772, "avg_output_tokens_per_task": 14077.236, "last_run_timestamp": "2026-10-10T11:21:32.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.7653059591836735, "accuracy_std_dev": 0.005416499999999991, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.02433298979591837, "avg_duration_per_task_ms": 682809.275510204, "avg_output_tokens_per_task": 13469.224489795919, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.7649122421052632, "accuracy_std_dev": 0.0039184999999999914, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.25060463157894736, "avg_duration_per_task_ms": 358896.81052631576, "avg_output_tokens_per_task": 6743.5684210526315, "last_run_timestamp": "2026-10-10T11:20:06.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7648808794642857, "accuracy_std_dev": 0.021171058001169644, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.10855453124999999, "avg_duration_per_task_ms": 356320.9129464286, "avg_output_tokens_per_task": 11844.839285714286, "last_run_timestamp": "2026-10-10T20:34:18.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7644445, "accuracy_std_dev": 0.016178189863207813, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.16990947999999997, "avg_duration_per_task_ms": 205513.85333333333, "avg_output_tokens_per_task": 11240.563333333334, "last_run_timestamp": "2026-10-10T18:54:48.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7644445, "accuracy_std_dev": 0.019876128745725773, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07212067999999999, "avg_duration_per_task_ms": 124793.37, "avg_output_tokens_per_task": 4621.306666666666, "last_run_timestamp": "2026-10-10T12:09:46.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.7644445, "accuracy_std_dev": 0.023622535799175632, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.050448443333333336, "avg_duration_per_task_ms": 138327.16, "avg_output_tokens_per_task": 4647.42, "last_run_timestamp": "2026-10-10T11:23:27.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02870791333333333, "avg_duration_per_task_ms": 1126711.7933333332, "avg_output_tokens_per_task": 60958.193333333336, "last_run_timestamp": "2026-10-10T12:23:30.561Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029735113333333334, "avg_duration_per_task_ms": 54092.806666666664, "avg_output_tokens_per_task": 4732.093333333333, "last_run_timestamp": "2026-10-03T08:37:59.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.764, "accuracy_std_dev": 0.02425793799151117, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.416801944, "avg_duration_per_task_ms": 97505.054, "avg_output_tokens_per_task": 5629.594, "last_run_timestamp": "2026-10-10T11:22:55.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.764, "accuracy_std_dev": 0.02515740668669966, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.23303213200000003, "avg_duration_per_task_ms": 355571.66, "avg_output_tokens_per_task": 17605.012, "last_run_timestamp": "2026-10-10T11:21:47.208Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7638094285714286, "accuracy_std_dev": 0.03917523513233161, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009541142857142856, "avg_duration_per_task_ms": 210820.91142857142, "avg_output_tokens_per_task": 15468.797142857144, "last_run_timestamp": "2026-10-10T18:57:39.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7638094285714286, "accuracy_std_dev": 0.03525091916068954, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.006661174285714286, "avg_duration_per_task_ms": 122379.59714285714, "avg_output_tokens_per_task": 4276.557142857143, "last_run_timestamp": "2026-10-10T12:05:49.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7637778000000002, "accuracy_std_dev": 0.022089610541911643, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.6385702406666667, "avg_duration_per_task_ms": 72802.16, "avg_output_tokens_per_task": 5665.454, "last_run_timestamp": "2026-10-10T11:34:56.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.76375, "accuracy_std_dev": 0.012849189342133631, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1187352025, "avg_duration_per_task_ms": 490603.2525, "avg_output_tokens_per_task": 21506.57, "last_run_timestamp": "2026-10-10T12:38:14.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7637037777777778, "accuracy_std_dev": 0.023749974515905763, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04213422666666667, "avg_duration_per_task_ms": 238270.13333333333, "avg_output_tokens_per_task": 12104.153333333334, "last_run_timestamp": "2026-10-10T19:15:26.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7634529596412556, "accuracy_std_dev": 0.02656307081059885, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.2075218811659193, "avg_duration_per_task_ms": 487178.55829596415, "avg_output_tokens_per_task": 20355.004484304933, "last_run_timestamp": "2026-10-10T13:15:05.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7634227449664429, "accuracy_std_dev": 0.018780119728052862, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.020238607382550334, "avg_duration_per_task_ms": 117111.80872483221, "avg_output_tokens_per_task": 14479.771812080537, "last_run_timestamp": "2026-10-10T11:09:17.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01259681, "avg_duration_per_task_ms": 657803.07, "avg_output_tokens_per_task": 25182.87, "last_run_timestamp": "2026-10-03T08:05:49.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.763333375, "accuracy_std_dev": 0.016666625002812498, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014786937500000001, "avg_duration_per_task_ms": 384155.3975, "avg_output_tokens_per_task": 15743.1525, "last_run_timestamp": "2026-10-10T19:26:17.283Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.763333375, "accuracy_std_dev": 0.019148607435904456, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09197625250000002, "avg_duration_per_task_ms": 241657.31, "avg_output_tokens_per_task": 16454.4975, "last_run_timestamp": "2026-10-10T19:19:35.243Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7633332142857143, "accuracy_std_dev": 0.017772848027975442, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.009256418571428572, "avg_duration_per_task_ms": 359533.8657142857, "avg_output_tokens_per_task": 14645.47, "last_run_timestamp": "2026-10-10T11:45:08.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06322074000000001, "avg_duration_per_task_ms": 968065, "avg_output_tokens_per_task": 43847.04, "last_run_timestamp": "2026-10-03T08:20:05.772Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.762963, "accuracy_std_dev": 0.033717927809664976, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04426393111111111, "avg_duration_per_task_ms": 210488.84444444443, "avg_output_tokens_per_task": 9459.097777777777, "last_run_timestamp": "2026-10-10T19:42:04.863Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7628572857142857, "accuracy_std_dev": 0.02869817284733489, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.14805788285714286, "avg_duration_per_task_ms": 317351.07714285713, "avg_output_tokens_per_task": 20174.72857142857, "last_run_timestamp": "2026-10-10T12:09:30.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7627551734693877, "accuracy_std_dev": 0.020230852904660254, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.025161864795918365, "avg_duration_per_task_ms": 402528.1301020408, "avg_output_tokens_per_task": 12698.543367346938, "last_run_timestamp": "2026-10-10T18:47:04.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7626666, "accuracy_std_dev": 0.03227647784749755, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.146569104, "avg_duration_per_task_ms": 373419.58, "avg_output_tokens_per_task": 24831.152, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7625640769230769, "accuracy_std_dev": 0.027739709669218682, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.5190420169230768, "avg_duration_per_task_ms": 219683.05076923076, "avg_output_tokens_per_task": 10161.08, "last_run_timestamp": "2026-10-10T11:39:24.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7621439748743719, "accuracy_std_dev": 0.01479216708895287, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.02848052763819095, "avg_duration_per_task_ms": 102090.86432160804, "avg_output_tokens_per_task": 17372.989949748742, "last_run_timestamp": "2026-10-10T11:32:54.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7620649764309765, "accuracy_std_dev": 0.008722071727647192, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.007020558922558921, "avg_duration_per_task_ms": 240106.04377104377, "avg_output_tokens_per_task": 12275.050505050505, "last_run_timestamp": "2026-08-29T16:33:29.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.76166675, "accuracy_std_dev": 0.021794590307860812, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06705333, "avg_duration_per_task_ms": 1165177.905, "avg_output_tokens_per_task": 61424.67, "last_run_timestamp": "2026-10-10T19:06:52.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.761666625, "accuracy_std_dev": 0.02533126362292996, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5473962499999999, "avg_duration_per_task_ms": 178661.2125, "avg_output_tokens_per_task": 10775.3575, "last_run_timestamp": "2026-10-10T11:09:36.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7611111666666667, "accuracy_std_dev": 0.022580443304008813, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13447535, "avg_duration_per_task_ms": 192048.67666666667, "avg_output_tokens_per_task": 8291.013333333334, "last_run_timestamp": "2026-10-10T14:25:49.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7610694385964911, "accuracy_std_dev": 0.023000953436759987, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.011744431077694234, "avg_duration_per_task_ms": 283947.4812030075, "avg_output_tokens_per_task": 13467.954887218046, "last_run_timestamp": "2026-10-10T18:43:43.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.76083325, "accuracy_std_dev": 0.048812266503282735, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0205383975, "avg_duration_per_task_ms": 237661.9975, "avg_output_tokens_per_task": 12406.4725, "last_run_timestamp": "2026-10-10T19:04:06.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7606667, "accuracy_std_dev": 0.02298790890033282, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.049680242, "avg_duration_per_task_ms": 131451.166, "avg_output_tokens_per_task": 4591.58, "last_run_timestamp": "2026-10-10T12:34:34.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7600001249999999, "accuracy_std_dev": 0.03349974295213884, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0116075025, "avg_duration_per_task_ms": 155751.2425, "avg_output_tokens_per_task": 14262.745, "last_run_timestamp": "2026-10-03T16:12:33.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7600001249999999, "accuracy_std_dev": 0.03282949216191708, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.087410225, "avg_duration_per_task_ms": 68616.515, "avg_output_tokens_per_task": 12187.7175, "last_run_timestamp": "2026-10-10T19:27:59.354Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.76, "accuracy_std_dev": 0.03144669210743797, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01082417, "avg_duration_per_task_ms": 1179559.2475, "avg_output_tokens_per_task": 12522.405, "last_run_timestamp": "2026-10-10T19:51:16.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.76, "accuracy_std_dev": 0.01989890611886173, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.006725167272727272, "avg_duration_per_task_ms": 258933.9818181818, "avg_output_tokens_per_task": 15286.843636363636, "last_run_timestamp": "2026-10-10T11:41:36.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.76, "accuracy_std_dev": 0.010886348913509402, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06629763999999999, "avg_duration_per_task_ms": 203915.74666666667, "avg_output_tokens_per_task": 10663.093333333334, "last_run_timestamp": "2026-10-10T18:57:23.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.76, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12531742999999998, "avg_duration_per_task_ms": 312784.77, "avg_output_tokens_per_task": 20756.6, "last_run_timestamp": "2026-10-03T08:29:42.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019761673333333334, "avg_duration_per_task_ms": 291193.25333333336, "avg_output_tokens_per_task": 17811.393333333333, "last_run_timestamp": "2026-10-10T11:23:23.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.76, "accuracy_std_dev": 0.01699689514293714, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021459479999999996, "avg_duration_per_task_ms": 117845.22, "avg_output_tokens_per_task": 8829.08, "last_run_timestamp": "2026-10-10T18:45:26.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.01914880327200632, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09582124500000001, "avg_duration_per_task_ms": 338140.68, "avg_output_tokens_per_task": 12863.7625, "last_run_timestamp": "2026-10-10T19:58:55.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.76, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.30033978, "avg_duration_per_task_ms": 256464.59, "avg_output_tokens_per_task": 10538.18, "last_run_timestamp": "2026-10-10T11:11:00.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07523408, "avg_duration_per_task_ms": 676626.08, "avg_output_tokens_per_task": 16285.62, "last_run_timestamp": "2026-10-03T08:15:42.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.76, "accuracy_std_dev": 0.015071811643668545, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05839733999999999, "avg_duration_per_task_ms": 223770.25333333333, "avg_output_tokens_per_task": 10458.826666666666, "last_run_timestamp": "2026-10-10T19:41:57.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7598205185185184, "accuracy_std_dev": 0.017395146832761534, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.1637978249158249, "avg_duration_per_task_ms": 852122.5521885522, "avg_output_tokens_per_task": 31760.62962962963, "last_run_timestamp": "2026-10-10T18:46:35.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7597053453815261, "accuracy_std_dev": 0.039111697810373804, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.11686768072289157, "avg_duration_per_task_ms": 385580.7168674699, "avg_output_tokens_per_task": 18897.180722891568, "last_run_timestamp": "2026-10-10T12:15:32.028Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.7593333, "accuracy_std_dev": 0.040381551894522326, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.19732587599999998, "avg_duration_per_task_ms": 103738.506, "avg_output_tokens_per_task": 7652.948, "last_run_timestamp": "2026-10-10T11:23:34.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7593333, "accuracy_std_dev": 0.016452043441773433, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.083344808, "avg_duration_per_task_ms": 106115.21, "avg_output_tokens_per_task": 23741.31, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7591521098265895, "accuracy_std_dev": 0.026787431210497644, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.022639277456647403, "avg_duration_per_task_ms": 278720.62716763007, "avg_output_tokens_per_task": 4187.803468208092, "last_run_timestamp": "2026-10-10T11:48:33.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7590477142857143, "accuracy_std_dev": 0.013997181448055748, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.24529245999999996, "avg_duration_per_task_ms": 151053.55142857143, "avg_output_tokens_per_task": 23224.54, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7590474285714286, "accuracy_std_dev": 0.01256277841035793, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03711470857142857, "avg_duration_per_task_ms": 81451.03428571428, "avg_output_tokens_per_task": 4616.405714285715, "last_run_timestamp": "2026-10-10T11:07:20.511Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.7588833197969542, "accuracy_std_dev": 0.017045916820458813, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.03293115228426395, "avg_duration_per_task_ms": 176423.84517766497, "avg_output_tokens_per_task": 16897.4923857868, "last_run_timestamp": "2026-10-10T11:29:01.096Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7586667, "accuracy_std_dev": 0.030521374661210805, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.20059342000000002, "avg_duration_per_task_ms": 127826.938, "avg_output_tokens_per_task": 7898.85, "last_run_timestamp": "2026-10-10T11:32:27.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.7586666, "accuracy_std_dev": 0.027776885675683684, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.548834584, "avg_duration_per_task_ms": 192613.822, "avg_output_tokens_per_task": 10699.43, "last_run_timestamp": "2026-10-10T11:37:23.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7586666, "accuracy_std_dev": 0.011469759275590758, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.065548452, "avg_duration_per_task_ms": 298543.66, "avg_output_tokens_per_task": 14030.988, "last_run_timestamp": "2026-10-10T11:18:50.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7585170300601203, "accuracy_std_dev": 0.026101515127670283, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.013800801603206412, "avg_duration_per_task_ms": 219369.76553106212, "avg_output_tokens_per_task": 16017.749498997997, "last_run_timestamp": "2026-10-10T11:53:37.852Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.40697005500000005, "avg_duration_per_task_ms": 402356.405, "avg_output_tokens_per_task": 35595.7, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.061411685, "avg_duration_per_task_ms": 71222.875, "avg_output_tokens_per_task": 4382.8, "last_run_timestamp": "2026-10-10T12:11:29.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7583331666666667, "accuracy_std_dev": 0.013844473427047428, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09591996333333334, "avg_duration_per_task_ms": 214673.9, "avg_output_tokens_per_task": 9689.54, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7582160060362173, "accuracy_std_dev": 0.027807515956122388, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.06698872032193161, "avg_duration_per_task_ms": 262502.1629778672, "avg_output_tokens_per_task": 11819.241448692153, "last_run_timestamp": "2026-10-10T11:04:01.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.7580952857142857, "accuracy_std_dev": 0.026358758071298158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08864299142857142, "avg_duration_per_task_ms": 293845.9, "avg_output_tokens_per_task": 12060.377142857144, "last_run_timestamp": "2026-10-10T11:33:11.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.758, "accuracy_std_dev": 0.03709447398198281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.079319272, "avg_duration_per_task_ms": 620750.224, "avg_output_tokens_per_task": 10895.356, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7579964141414142, "accuracy_std_dev": 0.023533015514113357, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20713992929292932, "avg_duration_per_task_ms": 664979.7727272727, "avg_output_tokens_per_task": 53099.67929292929, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7579564673366834, "accuracy_std_dev": 0.015677192643694233, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.011453532663316581, "avg_duration_per_task_ms": 323217.94974874373, "avg_output_tokens_per_task": 15027.713567839195, "last_run_timestamp": "2026-10-10T11:55:38.365Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7579487692307693, "accuracy_std_dev": 0.02358978428229489, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.027215110769230763, "avg_duration_per_task_ms": 172637.00615384616, "avg_output_tokens_per_task": 17994.436923076923, "last_run_timestamp": "2026-10-10T11:30:09.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.757778, "accuracy_std_dev": 0.03095182238038124, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06718082666666668, "avg_duration_per_task_ms": 287456.31333333335, "avg_output_tokens_per_task": 13536.3, "last_run_timestamp": "2026-10-10T12:17:59.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7577778333333334, "accuracy_std_dev": 0.013698760873605408, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19346874333333333, "avg_duration_per_task_ms": 319237.7366666667, "avg_output_tokens_per_task": 12183.71, "last_run_timestamp": "2026-10-10T11:32:01.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.02199916945907024, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.33451158000000003, "avg_duration_per_task_ms": 129909.81333333334, "avg_output_tokens_per_task": 7169.106666666667, "last_run_timestamp": "2026-10-10T11:07:11.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.04629468145358481, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12420428666666666, "avg_duration_per_task_ms": 116279.86, "avg_output_tokens_per_task": 19029.046666666665, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.35916456, "avg_duration_per_task_ms": 165675.65333333332, "avg_output_tokens_per_task": 12758.866666666667, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7574073333333333, "accuracy_std_dev": 0.015053593332564244, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.34690624666666664, "avg_duration_per_task_ms": 307425.4088888889, "avg_output_tokens_per_task": 10583.915555555555, "last_run_timestamp": "2026-10-10T11:25:46.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7573627951807229, "accuracy_std_dev": 0.03546880209832297, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.19399629919678715, "avg_duration_per_task_ms": 481017.5502008032, "avg_output_tokens_per_task": 28021.941767068274, "last_run_timestamp": "2026-10-10T19:09:38.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7571839109195402, "accuracy_std_dev": 0.02316220409317909, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.020125860632183912, "avg_duration_per_task_ms": 462996.8103448276, "avg_output_tokens_per_task": 17023.745689655174, "last_run_timestamp": "2026-10-10T11:45:16.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.757037, "accuracy_std_dev": 0.032827576317615806, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16180529777777777, "avg_duration_per_task_ms": 235632.8177777778, "avg_output_tokens_per_task": 9773.54, "last_run_timestamp": "2026-10-10T20:11:50.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7566667, "accuracy_std_dev": 0.028166280077603446, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.534804308, "avg_duration_per_task_ms": 188676.386, "avg_output_tokens_per_task": 10302.236, "last_run_timestamp": "2026-10-10T11:08:59.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.01875779344402985, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02229521, "avg_duration_per_task_ms": 131709.92666666667, "avg_output_tokens_per_task": 15842.956666666667, "last_run_timestamp": "2026-10-10T11:09:54.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.01990715486675305, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.0064209, "avg_duration_per_task_ms": 136204.27333333335, "avg_output_tokens_per_task": 4260.18, "last_run_timestamp": "2026-10-10T11:23:25.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.756551724137931, "accuracy_std_dev": 0.029588262127353878, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.2020337827586207, "avg_duration_per_task_ms": 119704.51379310346, "avg_output_tokens_per_task": 7806.491724137931, "last_run_timestamp": "2026-10-10T11:48:54.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.756538611018364, "accuracy_std_dev": 0.027065118181665127, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.08304781302170285, "avg_duration_per_task_ms": 142342.55425709515, "avg_output_tokens_per_task": 12749.095158597664, "last_run_timestamp": "2026-10-10T18:35:14.365Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7564913157894737, "accuracy_std_dev": 0.03429274061902074, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.19956802315789474, "avg_duration_per_task_ms": 102733.17894736842, "avg_output_tokens_per_task": 7681.796842105263, "last_run_timestamp": "2026-10-10T11:31:19.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7564102307692308, "accuracy_std_dev": 0.020358506100376164, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.026013500000000002, "avg_duration_per_task_ms": 296382.0492307692, "avg_output_tokens_per_task": 17325.696923076925, "last_run_timestamp": "2026-10-10T13:17:24.675Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7563351081871345, "accuracy_std_dev": 0.013338594901982123, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.033182684210526316, "avg_duration_per_task_ms": 2146298.769005848, "avg_output_tokens_per_task": 50213.23391812865, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7561907142857143, "accuracy_std_dev": 0.01846738376330619, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.14821341999999998, "avg_duration_per_task_ms": 147978.52, "avg_output_tokens_per_task": 20008.071428571428, "last_run_timestamp": "2026-10-10T11:08:09.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7561904285714286, "accuracy_std_dev": 0.02541274877951248, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07065733999999999, "avg_duration_per_task_ms": 161225.76571428572, "avg_output_tokens_per_task": 4721.2, "last_run_timestamp": "2026-10-10T11:58:35.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7561520067114095, "accuracy_std_dev": 0.03687511483673266, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.05932429530201341, "avg_duration_per_task_ms": 336709.1543624161, "avg_output_tokens_per_task": 47869.12080536913, "last_run_timestamp": "2026-10-10T18:58:53.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.756, "accuracy_std_dev": 0.036090468630928046, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.22400043066666664, "avg_duration_per_task_ms": 97447.95733333334, "avg_output_tokens_per_task": 7557.712666666666, "last_run_timestamp": "2026-10-10T11:27:47.581Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.75583325, "accuracy_std_dev": 0.024650453569610042, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012750845, "avg_duration_per_task_ms": 348532.645, "avg_output_tokens_per_task": 4259.125, "last_run_timestamp": "2026-10-10T11:53:43.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7557575454545454, "accuracy_std_dev": 0.021133944253321, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.022080854545454547, "avg_duration_per_task_ms": 128740.06909090909, "avg_output_tokens_per_task": 17525.272727272728, "last_run_timestamp": "2026-10-10T11:50:52.789Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7557575454545454, "accuracy_std_dev": 0.02365967382148038, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.022588227272727274, "avg_duration_per_task_ms": 122776.59454545454, "avg_output_tokens_per_task": 15090.52909090909, "last_run_timestamp": "2026-10-10T11:23:47.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7557079178082191, "accuracy_std_dev": 0.01607886717264481, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.2608585205479452, "avg_duration_per_task_ms": 125025.47260273973, "avg_output_tokens_per_task": 7152.054794520548, "last_run_timestamp": "2026-10-03T08:05:09.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7556891325301206, "accuracy_std_dev": 0.028686868714448425, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.05997379919678716, "avg_duration_per_task_ms": 208407.72289156626, "avg_output_tokens_per_task": 7587.301204819277, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7555803571428571, "accuracy_std_dev": 0.023882484930784247, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.044577171874999995, "avg_duration_per_task_ms": 507800.79910714284, "avg_output_tokens_per_task": 22983.540178571428, "last_run_timestamp": "2026-10-09T00:21:32.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.027460826666666667, "avg_duration_per_task_ms": 642340.9866666667, "avg_output_tokens_per_task": 17001.633333333335, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7555555, "accuracy_std_dev": 0.02819693530480457, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025840563333333334, "avg_duration_per_task_ms": 269041.9766666667, "avg_output_tokens_per_task": 13228.333333333334, "last_run_timestamp": "2026-10-10T18:35:19.634Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7554166250000001, "accuracy_std_dev": 0.041664622033979574, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1590986875, "avg_duration_per_task_ms": 337073.18, "avg_output_tokens_per_task": 22844.935, "last_run_timestamp": "2026-10-10T11:23:09.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7554165, "accuracy_std_dev": 0.031576074364144766, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0160830725, "avg_duration_per_task_ms": 75201.88, "avg_output_tokens_per_task": 4275.875, "last_run_timestamp": "2026-10-10T11:55:27.055Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7552381428571429, "accuracy_std_dev": 0.023289474817292592, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.00790055142857143, "avg_duration_per_task_ms": 177661.23142857142, "avg_output_tokens_per_task": 4430.751428571429, "last_run_timestamp": "2026-10-10T12:18:32.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7552380000000001, "accuracy_std_dev": 0.025872597329441617, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008791902857142856, "avg_duration_per_task_ms": 194073.0285714286, "avg_output_tokens_per_task": 4458.371428571429, "last_run_timestamp": "2026-10-10T11:41:33.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.7551516363636364, "accuracy_std_dev": 0.02221853890651896, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.24386067272727271, "avg_duration_per_task_ms": 305161.86363636365, "avg_output_tokens_per_task": 10007.39090909091, "last_run_timestamp": "2026-10-10T11:43:15.517Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.755027365630713, "accuracy_std_dev": 0.03406230364037725, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.19381868007312616, "avg_duration_per_task_ms": 569519.8921389397, "avg_output_tokens_per_task": 27223.491773308957, "last_run_timestamp": "2026-10-10T11:06:26.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.755000125, "accuracy_std_dev": 0.023033866975811414, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.19533066500000001, "avg_duration_per_task_ms": 121599.4775, "avg_output_tokens_per_task": 7673.8175, "last_run_timestamp": "2026-10-10T11:29:16.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.755, "accuracy_std_dev": 0.023033658947288434, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.018529425000000002, "avg_duration_per_task_ms": 140249.55, "avg_output_tokens_per_task": 4584.6825, "last_run_timestamp": "2026-10-10T11:55:21.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7548146666666667, "accuracy_std_dev": 0.03201517730007999, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09135532, "avg_duration_per_task_ms": 231928.58, "avg_output_tokens_per_task": 11085.384444444444, "last_run_timestamp": "2026-10-10T20:08:22.445Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7546667, "accuracy_std_dev": 0.02166412087784778, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.34589102000000005, "avg_duration_per_task_ms": 220634.988, "avg_output_tokens_per_task": 11962.75, "last_run_timestamp": "2026-10-10T11:51:26.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7544342825688073, "accuracy_std_dev": 0.02504935915479907, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.1000418, "avg_duration_per_task_ms": 796849.0642201835, "avg_output_tokens_per_task": 21296.05137614679, "last_run_timestamp": "2026-10-10T11:45:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.75416675, "accuracy_std_dev": 0.03169964729989123, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014154627500000003, "avg_duration_per_task_ms": 169480.67, "avg_output_tokens_per_task": 14972.985, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.75416675, "accuracy_std_dev": 0.030264911385753292, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0725496575, "avg_duration_per_task_ms": 303727.6275, "avg_output_tokens_per_task": 11490.4825, "last_run_timestamp": "2026-10-10T21:35:36.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.75416675, "accuracy_std_dev": 0.012219036835507957, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1638202875, "avg_duration_per_task_ms": 453197.5425, "avg_output_tokens_per_task": 9528.3525, "last_run_timestamp": "2026-10-03T16:16:10.656Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.7540001, "accuracy_std_dev": 0.032584956993833826, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.020762876000000003, "avg_duration_per_task_ms": 187591.642, "avg_output_tokens_per_task": 14436.292, "last_run_timestamp": "2026-10-10T11:41:11.074Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.753914899328859, "accuracy_std_dev": 0.040895046741628756, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.07723322147651007, "avg_duration_per_task_ms": 229349.62863534674, "avg_output_tokens_per_task": 14123.306487695749, "last_run_timestamp": "2026-10-03T08:50:19.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7537931724137933, "accuracy_std_dev": 0.024380773394523443, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.3712446551724138, "avg_duration_per_task_ms": 167809.39172413794, "avg_output_tokens_per_task": 11853.664827586206, "last_run_timestamp": "2026-10-10T11:36:16.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7533785608108108, "accuracy_std_dev": 0.03423001508788255, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.056939652027027034, "avg_duration_per_task_ms": 407763.0641891892, "avg_output_tokens_per_task": 12744.533783783783, "last_run_timestamp": "2026-10-10T11:23:27.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.7533336666666666, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3148796866666667, "avg_duration_per_task_ms": 281526.8, "avg_output_tokens_per_task": 9656.493333333334, "last_run_timestamp": "2026-10-10T11:41:13.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7533335, "accuracy_std_dev": 0.053333500000000034, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00958291, "avg_duration_per_task_ms": 393540.68, "avg_output_tokens_per_task": 4161.75, "last_run_timestamp": "2026-10-10T11:31:07.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11451421333333334, "avg_duration_per_task_ms": 146387.56666666668, "avg_output_tokens_per_task": 17546.113333333335, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.29158895333333334, "avg_duration_per_task_ms": 171961.5, "avg_output_tokens_per_task": 10696.2, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.03678447069076843, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08935323333333334, "avg_duration_per_task_ms": 248145.7288888889, "avg_output_tokens_per_task": 12283.197777777777, "last_run_timestamp": "2026-10-10T11:25:27.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7533332857142857, "accuracy_std_dev": 0.017089883763830138, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1783802742857143, "avg_duration_per_task_ms": 316437.2057142857, "avg_output_tokens_per_task": 10826.988571428572, "last_run_timestamp": "2026-10-10T11:07:50.254Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7533332500000001, "accuracy_std_dev": 0.026246544747594865, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.34973291500000003, "avg_duration_per_task_ms": 104206.33, "avg_output_tokens_per_task": 7119.515, "last_run_timestamp": "2026-10-10T11:23:32.977Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7533332500000001, "accuracy_std_dev": 0.027487471895165275, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1073622525, "avg_duration_per_task_ms": 273662.3775, "avg_output_tokens_per_task": 12208.0525, "last_run_timestamp": "2026-10-10T11:27:45.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7533332222222222, "accuracy_std_dev": 0.02591535128399458, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013880666666666666, "avg_duration_per_task_ms": 210103.7488888889, "avg_output_tokens_per_task": 14896.853333333333, "last_run_timestamp": "2026-10-10T18:54:17.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7533332, "accuracy_std_dev": 0.02732525083800698, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011721984, "avg_duration_per_task_ms": 506375.232, "avg_output_tokens_per_task": 6019.948, "last_run_timestamp": "2026-10-10T11:38:18.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0119157, "avg_duration_per_task_ms": 95782.12, "avg_output_tokens_per_task": 11167.88, "last_run_timestamp": "2026-10-10T11:10:14.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.032659863237109024, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12209446666666665, "avg_duration_per_task_ms": 147223.81333333332, "avg_output_tokens_per_task": 19831.306666666667, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2515843, "avg_duration_per_task_ms": 421006.06, "avg_output_tokens_per_task": 32340.48, "last_run_timestamp": "2026-10-01T08:00:24.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0196027, "avg_duration_per_task_ms": 76385.02, "avg_output_tokens_per_task": 9950, "last_run_timestamp": "2026-07-18T08:04:02.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7526158878923768, "accuracy_std_dev": 0.04664293923964891, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.07574351569506728, "avg_duration_per_task_ms": 360044.43497757846, "avg_output_tokens_per_task": 12747.226457399103, "last_run_timestamp": "2026-10-10T19:37:33.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.03021638651133634, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5157013133333334, "avg_duration_per_task_ms": 178548.1511111111, "avg_output_tokens_per_task": 7818.084444444445, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7525, "accuracy_std_dev": 0.03044809791678292, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021822374999999998, "avg_duration_per_task_ms": 485502.945, "avg_output_tokens_per_task": 14781.975, "last_run_timestamp": "2026-10-10T18:37:34.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7522221666666666, "accuracy_std_dev": 0.01511278994115765, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.44058496666666663, "avg_duration_per_task_ms": 458389.25, "avg_output_tokens_per_task": 35681.92333333333, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.752, "accuracy_std_dev": 0.022469515152757562, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019928192, "avg_duration_per_task_ms": 74879.332, "avg_output_tokens_per_task": 4253.16, "last_run_timestamp": "2026-10-10T11:24:18.065Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.7517005306122448, "accuracy_std_dev": 0.012796557796532647, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.09461716326530611, "avg_duration_per_task_ms": 209290.9224489796, "avg_output_tokens_per_task": 7253.518367346939, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7515597513368985, "accuracy_std_dev": 0.021149574595143886, "run_count": 15, "total_questions": 748, "avg_cost_per_task_usd": 0.00707783155080214, "avg_duration_per_task_ms": 201238.5909090909, "avg_output_tokens_per_task": 13061.36898395722, "last_run_timestamp": "2026-10-10T11:51:08.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7514815555555555, "accuracy_std_dev": 0.0333005142973669, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07395462222222221, "avg_duration_per_task_ms": 378942.0577777778, "avg_output_tokens_per_task": 9157.224444444444, "last_run_timestamp": "2026-10-10T18:30:31.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.7513333999999999, "accuracy_std_dev": 0.030521588717496353, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.062658464, "avg_duration_per_task_ms": 1700876.012, "avg_output_tokens_per_task": 8868.604, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7511113333333332, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48754586, "avg_duration_per_task_ms": 141651.29333333333, "avg_output_tokens_per_task": 7913.286666666667, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.751111, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03708808, "avg_duration_per_task_ms": 176459.57333333333, "avg_output_tokens_per_task": 25948.413333333334, "last_run_timestamp": "2026-10-10T11:21:01.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.750833375, "accuracy_std_dev": 0.028467763202162117, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.022724927500000002, "avg_duration_per_task_ms": 216195.8375, "avg_output_tokens_per_task": 13169.8975, "last_run_timestamp": "2026-10-10T18:54:11.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7506667, "accuracy_std_dev": 0.026365013855676238, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.49418161999999993, "avg_duration_per_task_ms": 195069.658, "avg_output_tokens_per_task": 10103.428, "last_run_timestamp": "2026-10-10T11:51:44.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7505569242761692, "accuracy_std_dev": 0.02142411426983423, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.028247770601336302, "avg_duration_per_task_ms": 598851.2494432072, "avg_output_tokens_per_task": 22337.24053452116, "last_run_timestamp": "2026-10-10T19:09:44.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7505555, "accuracy_std_dev": 0.013181957837008396, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.36349930333333336, "avg_duration_per_task_ms": 169671.99166666667, "avg_output_tokens_per_task": 7150.9766666666665, "last_run_timestamp": "2026-10-10T11:07:06.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7503702222222223, "accuracy_std_dev": 0.036799392403733415, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07642833777777779, "avg_duration_per_task_ms": 230421.34666666668, "avg_output_tokens_per_task": 9496.562222222223, "last_run_timestamp": "2026-10-10T19:41:42.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7503701111111112, "accuracy_std_dev": 0.02516343723405248, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0877466288888889, "avg_duration_per_task_ms": 251071.52, "avg_output_tokens_per_task": 9774.766666666666, "last_run_timestamp": "2026-10-10T20:31:30.609Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7503029090909091, "accuracy_std_dev": 0.02076579266903998, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03820286181818182, "avg_duration_per_task_ms": 292584.13636363635, "avg_output_tokens_per_task": 23337.65090909091, "last_run_timestamp": "2026-10-10T18:44:59.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7500001999999999, "accuracy_std_dev": 0.026832766040048864, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.015060424, "avg_duration_per_task_ms": 269962.708, "avg_output_tokens_per_task": 16418.532, "last_run_timestamp": "2026-10-10T11:41:33.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01247142, "avg_duration_per_task_ms": 96762.37, "avg_output_tokens_per_task": 10932.5, "last_run_timestamp": "2026-10-10T11:07:19.680Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.75, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05066206, "avg_duration_per_task_ms": 368063.6, "avg_output_tokens_per_task": 8379.6, "last_run_timestamp": "2026-07-25T08:07:26.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.75, "accuracy_std_dev": 0.04384306609373936, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.50077502, "avg_duration_per_task_ms": 139265.845, "avg_output_tokens_per_task": 7986.47, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7498134675615213, "accuracy_std_dev": 0.030550014400594683, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.06635837807606264, "avg_duration_per_task_ms": 426889.067114094, "avg_output_tokens_per_task": 10337.557046979866, "last_run_timestamp": "2026-10-10T18:57:24.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.7496296666666666, "accuracy_std_dev": 0.025553012220784384, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.32866541777777775, "avg_duration_per_task_ms": 573042.4666666667, "avg_output_tokens_per_task": 26324.751111111113, "last_run_timestamp": "2026-10-10T18:29:59.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7495746530612246, "accuracy_std_dev": 0.021234638556127484, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.014195257653061224, "avg_duration_per_task_ms": 56974.566326530614, "avg_output_tokens_per_task": 3915.466836734694, "last_run_timestamp": "2026-10-10T11:44:02.286Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7495237142857143, "accuracy_std_dev": 0.027338579222746164, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.17267997714285713, "avg_duration_per_task_ms": 322644.1828571429, "avg_output_tokens_per_task": 11143.974285714286, "last_run_timestamp": "2026-10-10T11:20:38.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7493333, "accuracy_std_dev": 0.028237051310467966, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.5616704749999999, "avg_duration_per_task_ms": 202133.878, "avg_output_tokens_per_task": 9880.147, "last_run_timestamp": "2026-10-10T11:58:23.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7492560625, "accuracy_std_dev": 0.02625846130626927, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.005948455357142857, "avg_duration_per_task_ms": 261080.47767857142, "avg_output_tokens_per_task": 12566.60267857143, "last_run_timestamp": "2026-10-10T18:32:35.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7491665000000001, "accuracy_std_dev": 0.022095128925851498, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11723742000000001, "avg_duration_per_task_ms": 336463.235, "avg_output_tokens_per_task": 9787.0025, "last_run_timestamp": "2026-10-10T19:45:51.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7490475714285715, "accuracy_std_dev": 0.017155993645346933, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034581794285714285, "avg_duration_per_task_ms": 410319.02857142856, "avg_output_tokens_per_task": 9408.788571428571, "last_run_timestamp": "2026-10-09T02:05:38.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7490367167630058, "accuracy_std_dev": 0.02179500605877504, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.011687421965317918, "avg_duration_per_task_ms": 205541.59826589594, "avg_output_tokens_per_task": 4094.841040462428, "last_run_timestamp": "2026-10-03T08:38:03.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7488888333333333, "accuracy_std_dev": 0.02948100036982387, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09071532, "avg_duration_per_task_ms": 524416.5966666667, "avg_output_tokens_per_task": 12656.81, "last_run_timestamp": "2026-10-10T11:02:27.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7488737702702704, "accuracy_std_dev": 0.035681240826082414, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.3075426081081082, "avg_duration_per_task_ms": 149494.7972972973, "avg_output_tokens_per_task": 20582.41554054054, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7487180769230769, "accuracy_std_dev": 0.026458784394024375, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.35342018461538466, "avg_duration_per_task_ms": 231535.48615384614, "avg_output_tokens_per_task": 10881.203076923077, "last_run_timestamp": "2026-10-10T11:05:56.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7486667, "accuracy_std_dev": 0.030265335507970156, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.548401042, "avg_duration_per_task_ms": 183718.584, "avg_output_tokens_per_task": 10523.854, "last_run_timestamp": "2026-10-10T11:37:47.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7486667, "accuracy_std_dev": 0.021509684642272205, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.32253603399999997, "avg_duration_per_task_ms": 198893.474, "avg_output_tokens_per_task": 11527.52, "last_run_timestamp": "2026-10-10T11:52:35.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.748484909090909, "accuracy_std_dev": 0.0478288811635331, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.13337880545454545, "avg_duration_per_task_ms": 874645.5763636364, "avg_output_tokens_per_task": 32352.327272727274, "last_run_timestamp": "2026-10-10T19:56:25.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7484848181818182, "accuracy_std_dev": 0.0229345089085429, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009084052727272729, "avg_duration_per_task_ms": 248840.61454545453, "avg_output_tokens_per_task": 4233.045454545455, "last_run_timestamp": "2026-10-10T11:56:23.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7483335, "accuracy_std_dev": 0.05644579011981318, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02282431, "avg_duration_per_task_ms": 395794.725, "avg_output_tokens_per_task": 15096.88, "last_run_timestamp": "2026-10-10T12:00:54.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.748333375, "accuracy_std_dev": 0.026822334704204524, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.34900722500000003, "avg_duration_per_task_ms": 202391.0825, "avg_output_tokens_per_task": 11926.3975, "last_run_timestamp": "2026-10-10T11:49:25.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7483132064777328, "accuracy_std_dev": 0.03485977243528707, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.10924008502024292, "avg_duration_per_task_ms": 442802.26720647776, "avg_output_tokens_per_task": 28877.655870445345, "last_run_timestamp": "2026-10-10T11:44:46.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.748, "accuracy_std_dev": 0.029333454545961698, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07858999599999998, "avg_duration_per_task_ms": 166655.096, "avg_output_tokens_per_task": 8640.336, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.015112569375817262, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01356969, "avg_duration_per_task_ms": 102348.01, "avg_output_tokens_per_task": 11130.603333333333, "last_run_timestamp": "2026-10-10T11:22:10.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7476922307692307, "accuracy_std_dev": 0.027029079235729832, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.33929932, "avg_duration_per_task_ms": 209258.88923076924, "avg_output_tokens_per_task": 11836.412307692308, "last_run_timestamp": "2026-10-10T11:52:21.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7473333, "accuracy_std_dev": 0.02866662093114572, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.520711906, "avg_duration_per_task_ms": 226639.988, "avg_output_tokens_per_task": 10065.72, "last_run_timestamp": "2026-10-10T11:53:44.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7472221666666666, "accuracy_std_dev": 0.011613686586906374, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17439429333333334, "avg_duration_per_task_ms": 161420.59666666668, "avg_output_tokens_per_task": 12499.9, "last_run_timestamp": "2026-10-10T11:19:48.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7470688291457286, "accuracy_std_dev": 0.06803424962002137, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.2510165728643216, "avg_duration_per_task_ms": 432358.3894472362, "avg_output_tokens_per_task": 28211.77135678392, "last_run_timestamp": "2026-10-10T18:34:57.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.7466670000000001, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16298501999999998, "avg_duration_per_task_ms": 1681916.16, "avg_output_tokens_per_task": 38161.67, "last_run_timestamp": "2026-10-10T12:00:26.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02836242, "avg_duration_per_task_ms": 881974.84, "avg_output_tokens_per_task": 13870.46, "last_run_timestamp": "2026-07-18T08:03:38.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02949668, "avg_duration_per_task_ms": 109584.32, "avg_output_tokens_per_task": 40655.62, "last_run_timestamp": "2026-10-10T18:33:47.508Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02698528, "avg_duration_per_task_ms": 147907.84, "avg_output_tokens_per_task": 11065.38, "last_run_timestamp": "2026-07-18T08:04:04.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.38669936, "avg_duration_per_task_ms": 338232.82, "avg_output_tokens_per_task": 9439.24, "last_run_timestamp": "2026-07-18T08:04:09.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7466667142857143, "accuracy_std_dev": 0.030446278476379096, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007662862857142856, "avg_duration_per_task_ms": 184997.26571428572, "avg_output_tokens_per_task": 4375.4742857142855, "last_run_timestamp": "2026-10-10T11:52:30.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7466667, "accuracy_std_dev": 0.025820018075322887, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.085612296, "avg_duration_per_task_ms": 852424.342, "avg_output_tokens_per_task": 36599.922, "last_run_timestamp": "2026-10-10T19:38:18.068Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.027034372269390698, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015178880000000002, "avg_duration_per_task_ms": 145373.45333333334, "avg_output_tokens_per_task": 16435.604444444445, "last_run_timestamp": "2026-10-03T08:25:27.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.023726809342082992, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21412848666666662, "avg_duration_per_task_ms": 142371.99333333335, "avg_output_tokens_per_task": 22756.126666666667, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18399340666666666, "avg_duration_per_task_ms": 116870.40666666666, "avg_output_tokens_per_task": 5117.2266666666665, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.746666625, "accuracy_std_dev": 0.01632993162062767, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.00887094, "avg_duration_per_task_ms": 153911.1775, "avg_output_tokens_per_task": 13256.88, "last_run_timestamp": "2026-10-10T18:34:58.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01003968, "avg_duration_per_task_ms": 293152.56, "avg_output_tokens_per_task": 17237.02, "last_run_timestamp": "2026-10-10T11:52:52.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7465427067238912, "accuracy_std_dev": 0.015385569261983973, "run_count": 14, "total_questions": 699, "avg_cost_per_task_usd": 0.048925171673819746, "avg_duration_per_task_ms": 263945.0042918455, "avg_output_tokens_per_task": 28450.080114449214, "last_run_timestamp": "2026-10-05T17:03:10.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.7465276875000001, "accuracy_std_dev": 0.001809000000000005, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.03102420833333333, "avg_duration_per_task_ms": 572971.2916666666, "avg_output_tokens_per_task": 10510.135416666666, "last_run_timestamp": "2026-10-10T12:09:56.410Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7462797633928571, "accuracy_std_dev": 0.024545314608085133, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.03578149553571429, "avg_duration_per_task_ms": 601980.8191964285, "avg_output_tokens_per_task": 15021.482142857143, "last_run_timestamp": "2026-10-10T19:19:24.410Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7459258888888889, "accuracy_std_dev": 0.029555403324169362, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07188528888888888, "avg_duration_per_task_ms": 265497.6822222222, "avg_output_tokens_per_task": 9840.462222222222, "last_run_timestamp": "2026-10-10T20:32:33.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7459258888888889, "accuracy_std_dev": 0.038160653111241984, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11357590222222222, "avg_duration_per_task_ms": 276977.56444444443, "avg_output_tokens_per_task": 9323.804444444444, "last_run_timestamp": "2026-10-10T19:45:37.580Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.74583325, "accuracy_std_dev": 0.030810710170125915, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11509349499999999, "avg_duration_per_task_ms": 368700.52, "avg_output_tokens_per_task": 8722.7075, "last_run_timestamp": "2026-10-10T20:05:26.252Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.7455555, "accuracy_std_dev": 0.03536397169318889, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04841413333333334, "avg_duration_per_task_ms": 491827.81, "avg_output_tokens_per_task": 9312.496666666666, "last_run_timestamp": "2026-10-10T19:27:19.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7454953513513514, "accuracy_std_dev": 0.009666043692569685, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.27390454729729735, "avg_duration_per_task_ms": 475940.8445945946, "avg_output_tokens_per_task": 13709.68918918919, "last_run_timestamp": "2026-10-10T11:24:20.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7453938040201006, "accuracy_std_dev": 0.03820312878139041, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.016367582914572864, "avg_duration_per_task_ms": 153008.0201005025, "avg_output_tokens_per_task": 3923.0502512562816, "last_run_timestamp": "2026-10-10T11:50:28.035Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7453333, "accuracy_std_dev": 0.036246163322619415, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.030892698000000003, "avg_duration_per_task_ms": 266107.998, "avg_output_tokens_per_task": 17044.59, "last_run_timestamp": "2026-10-10T11:29:00.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.02325970157818395, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07844153111111112, "avg_duration_per_task_ms": 265823.6488888889, "avg_output_tokens_per_task": 11310.011111111111, "last_run_timestamp": "2026-10-10T22:19:57.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7450001666666666, "accuracy_std_dev": 0.03916974692462142, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.025603288333333335, "avg_duration_per_task_ms": 399238.37333333335, "avg_output_tokens_per_task": 15346.756666666666, "last_run_timestamp": "2026-10-10T13:03:25.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7449999642857142, "accuracy_std_dev": 0.03317215855576367, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.37590654428571424, "avg_duration_per_task_ms": 201970.09785714286, "avg_output_tokens_per_task": 11978.649285714286, "last_run_timestamp": "2026-10-10T11:56:02.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7447618571428571, "accuracy_std_dev": 0.016218418466392492, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.013583602857142859, "avg_duration_per_task_ms": 386744.47428571427, "avg_output_tokens_per_task": 16391.625714285714, "last_run_timestamp": "2026-10-10T11:53:49.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7447618571428571, "accuracy_std_dev": 0.01621859463198753, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05547540571428572, "avg_duration_per_task_ms": 220891.45428571428, "avg_output_tokens_per_task": 9176.125714285714, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7447618571428571, "accuracy_std_dev": 0.026358618706647914, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008384540000000001, "avg_duration_per_task_ms": 237385.73142857142, "avg_output_tokens_per_task": 5177.305714285714, "last_run_timestamp": "2026-10-10T12:19:01.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2319787333333332, "avg_duration_per_task_ms": 98531.56, "avg_output_tokens_per_task": 5524.42, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.03499547766339085, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028980353333333334, "avg_duration_per_task_ms": 164219.27333333335, "avg_output_tokens_per_task": 9416.14, "last_run_timestamp": "2026-10-10T11:38:09.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.74416675, "accuracy_std_dev": 0.034146227627477405, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.127937195, "avg_duration_per_task_ms": 400891.875, "avg_output_tokens_per_task": 11822.0075, "last_run_timestamp": "2026-10-10T20:11:03.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.74416675, "accuracy_std_dev": 0.006400770358909954, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08563862, "avg_duration_per_task_ms": 309336.805, "avg_output_tokens_per_task": 9480.84, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7439999, "accuracy_std_dev": 0.022548769862012447, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014404088000000002, "avg_duration_per_task_ms": 119393.714, "avg_output_tokens_per_task": 15524.802, "last_run_timestamp": "2026-10-10T11:03:23.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7433335, "accuracy_std_dev": 0.023333285714832345, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.353090755, "avg_duration_per_task_ms": 88187.84, "avg_output_tokens_per_task": 6970.8425, "last_run_timestamp": "2026-10-10T11:20:56.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.04741472657718158, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08355239, "avg_duration_per_task_ms": 182015.24166666667, "avg_output_tokens_per_task": 8140.348333333333, "last_run_timestamp": "2026-10-10T12:46:20.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.742963, "accuracy_std_dev": 0.05087977976673161, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0923280022222222, "avg_duration_per_task_ms": 241983.94666666666, "avg_output_tokens_per_task": 11240.468888888889, "last_run_timestamp": "2026-10-10T20:09:18.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.742962888888889, "accuracy_std_dev": 0.04298856070177132, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009493242222222222, "avg_duration_per_task_ms": 317851.38666666666, "avg_output_tokens_per_task": 15469.368888888888, "last_run_timestamp": "2026-10-10T11:49:09.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.742962888888889, "accuracy_std_dev": 0.04098905672234545, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04189076222222222, "avg_duration_per_task_ms": 206004.44666666666, "avg_output_tokens_per_task": 19717.306666666667, "last_run_timestamp": "2026-10-10T11:33:22.065Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.74291675, "accuracy_std_dev": 0.0298810059057171, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0853285525, "avg_duration_per_task_ms": 188726.3925, "avg_output_tokens_per_task": 10064.6375, "last_run_timestamp": "2026-10-10T19:51:36.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7426966853932584, "accuracy_std_dev": 0.023739075903780084, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.022410094382022478, "avg_duration_per_task_ms": 289993.2921348315, "avg_output_tokens_per_task": 16658.757303370785, "last_run_timestamp": "2026-10-10T11:23:23.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.021746139377829814, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024568152000000003, "avg_duration_per_task_ms": 145960.132, "avg_output_tokens_per_task": 17370.136, "last_run_timestamp": "2026-10-10T11:10:23.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7425, "accuracy_std_dev": 0.020258742970628775, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.010828945, "avg_duration_per_task_ms": 216783.6, "avg_output_tokens_per_task": 14322.1425, "last_run_timestamp": "2026-10-03T16:11:33.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7423934827586207, "accuracy_std_dev": 0.04075671767696707, "run_count": 10, "total_questions": 493, "avg_cost_per_task_usd": 0.11995382961460448, "avg_duration_per_task_ms": 491199.4949290061, "avg_output_tokens_per_task": 23541.371196754564, "last_run_timestamp": "2026-10-03T08:18:33.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.742293063583815, "accuracy_std_dev": 0.017156108636605785, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.019260367052023122, "avg_duration_per_task_ms": 95216.28323699422, "avg_output_tokens_per_task": 4012.3959537572255, "last_run_timestamp": "2026-10-10T11:31:00.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.006285707880227596, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.38184122000000004, "avg_duration_per_task_ms": 149071.25333333333, "avg_output_tokens_per_task": 12794.173333333334, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0028365266666666666, "avg_duration_per_task_ms": 68565.14666666667, "avg_output_tokens_per_task": 11675.026666666667, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.74222225, "accuracy_std_dev": 0.028458392848522442, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023521068333333332, "avg_duration_per_task_ms": 166328.99, "avg_output_tokens_per_task": 7628.006666666667, "last_run_timestamp": "2026-10-10T18:45:14.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.742222, "accuracy_std_dev": 0.015713539024251252, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01313277, "avg_duration_per_task_ms": 70981.14, "avg_output_tokens_per_task": 10941.743333333334, "last_run_timestamp": "2026-10-10T11:35:03.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.742222, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.044708606666666664, "avg_duration_per_task_ms": 149318.53333333333, "avg_output_tokens_per_task": 38342.77333333333, "last_run_timestamp": "2026-10-10T11:04:02.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7422219999999999, "accuracy_std_dev": 0.036243279496572404, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06723134, "avg_duration_per_task_ms": 1036437.88, "avg_output_tokens_per_task": 42501.62, "last_run_timestamp": "2026-10-10T13:56:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7421700402684563, "accuracy_std_dev": 0.02926487036106602, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.06028264093959732, "avg_duration_per_task_ms": 213032.25167785236, "avg_output_tokens_per_task": 32783.35906040268, "last_run_timestamp": "2026-10-10T11:29:57.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7421122716049383, "accuracy_std_dev": 0.04949233809146621, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.07618280658436215, "avg_duration_per_task_ms": 605963.6255144033, "avg_output_tokens_per_task": 7686.909465020576, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7420002, "accuracy_std_dev": 0.025612585448564165, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.015137903999999999, "avg_duration_per_task_ms": 534442.24, "avg_output_tokens_per_task": 22548.816, "last_run_timestamp": "2026-10-10T11:30:00.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.7420002, "accuracy_std_dev": 0.031874700521887275, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014078051999999999, "avg_duration_per_task_ms": 39974.624, "avg_output_tokens_per_task": 3995.512, "last_run_timestamp": "2026-10-10T11:44:25.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.7420000999999999, "accuracy_std_dev": 0.01606935109112998, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.015730394, "avg_duration_per_task_ms": 180471.802, "avg_output_tokens_per_task": 4250.502, "last_run_timestamp": "2026-10-10T11:54:55.212Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7419048571428571, "accuracy_std_dev": 0.03181571083343471, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07701604857142856, "avg_duration_per_task_ms": 557828.82, "avg_output_tokens_per_task": 11690.26, "last_run_timestamp": "2026-10-03T16:12:13.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7417787623318386, "accuracy_std_dev": 0.020933986849907486, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.11384247085201793, "avg_duration_per_task_ms": 107222.65470852017, "avg_output_tokens_per_task": 10059.766816143498, "last_run_timestamp": "2026-10-10T18:37:34.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7416667499999999, "accuracy_std_dev": 0.02843115955140591, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.29442077, "avg_duration_per_task_ms": 504048.0925, "avg_output_tokens_per_task": 9618.08, "last_run_timestamp": "2026-10-10T21:46:04.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.7416665, "accuracy_std_dev": 0.011902170673872925, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012876185, "avg_duration_per_task_ms": 78684.405, "avg_output_tokens_per_task": 10797.005, "last_run_timestamp": "2026-10-10T11:31:34.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7416665, "accuracy_std_dev": 0.008660350267165872, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.34675104999999995, "avg_duration_per_task_ms": 129028.095, "avg_output_tokens_per_task": 7166.795, "last_run_timestamp": "2026-10-10T11:05:10.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7416106107382551, "accuracy_std_dev": 0.01944857756081121, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04701292617449664, "avg_duration_per_task_ms": 840147.7315436242, "avg_output_tokens_per_task": 14235.224832214764, "last_run_timestamp": "2026-10-10T19:13:25.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7414813333333333, "accuracy_std_dev": 0.021029672681977533, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07095983555555556, "avg_duration_per_task_ms": 335910.37111111113, "avg_output_tokens_per_task": 12802.966666666667, "last_run_timestamp": "2026-10-10T20:35:57.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7412059597989951, "accuracy_std_dev": 0.030598471473006233, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.09609109798994975, "avg_duration_per_task_ms": 367126.6331658292, "avg_output_tokens_per_task": 11257.263819095477, "last_run_timestamp": "2026-10-10T19:52:16.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.741147673992674, "accuracy_std_dev": 0.022891877368502254, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.14074178937728937, "avg_duration_per_task_ms": 591418.4395604396, "avg_output_tokens_per_task": 27023.639194139196, "last_run_timestamp": "2026-10-10T19:09:43.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7410256153846154, "accuracy_std_dev": 0.029911083221231823, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.06005055230769231, "avg_duration_per_task_ms": 271297.1076923077, "avg_output_tokens_per_task": 9692.147692307692, "last_run_timestamp": "2026-10-10T11:37:01.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7407407777777778, "accuracy_std_dev": 0.0330273719470441, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1090063111111111, "avg_duration_per_task_ms": 324070.6777777778, "avg_output_tokens_per_task": 18436.48, "last_run_timestamp": "2026-10-10T13:19:36.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.7407407777777778, "accuracy_std_dev": 0.03670968370812553, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.046896531111111105, "avg_duration_per_task_ms": 552618.62, "avg_output_tokens_per_task": 9268.293333333333, "last_run_timestamp": "2026-10-10T22:42:55.824Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7403703333333334, "accuracy_std_dev": 0.030325141571529955, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3409674066666667, "avg_duration_per_task_ms": 128710.22, "avg_output_tokens_per_task": 5246.528888888889, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7402943534136547, "accuracy_std_dev": 0.02302499266101948, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.06053981927710843, "avg_duration_per_task_ms": 161020.49799196786, "avg_output_tokens_per_task": 6902.349397590361, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400002, "accuracy_std_dev": 0.01264900523203309, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034869184, "avg_duration_per_task_ms": 129959.044, "avg_output_tokens_per_task": 16349.368, "last_run_timestamp": "2026-10-03T08:21:01.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7400000909090909, "accuracy_std_dev": 0.025584014897284242, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0668473418181818, "avg_duration_per_task_ms": 146402.66363636364, "avg_output_tokens_per_task": 9276.552727272727, "last_run_timestamp": "2026-10-10T18:47:02.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2696536, "avg_duration_per_task_ms": 310259.6, "avg_output_tokens_per_task": 11435.58, "last_run_timestamp": "2026-07-18T08:04:29.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05110574, "avg_duration_per_task_ms": 158677.44, "avg_output_tokens_per_task": 8249.06, "last_run_timestamp": "2026-07-18T16:00:09.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.74, "accuracy_std_dev": 0.044555500139339316, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.0935073933333332, "avg_duration_per_task_ms": 295552.94, "avg_output_tokens_per_task": 22860.126666666667, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.74, "accuracy_std_dev": 0.01805352125394564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.22917437000000002, "avg_duration_per_task_ms": 262336.34, "avg_output_tokens_per_task": 12064.356666666667, "last_run_timestamp": "2026-10-10T11:08:41.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02561014, "avg_duration_per_task_ms": 147210.62, "avg_output_tokens_per_task": 10787.18, "last_run_timestamp": "2026-07-18T08:04:42.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.74, "accuracy_std_dev": 0.0354338036978815, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09767998, "avg_duration_per_task_ms": 450597.1575, "avg_output_tokens_per_task": 9690.325, "last_run_timestamp": "2026-10-03T16:05:51.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12237688000000001, "avg_duration_per_task_ms": 90244.42, "avg_output_tokens_per_task": 9936.14, "last_run_timestamp": "2026-07-18T08:04:45.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03319066, "avg_duration_per_task_ms": 77162.14, "avg_output_tokens_per_task": 4453.24, "last_run_timestamp": "2026-10-04T03:00:16.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7399998571428572, "accuracy_std_dev": 0.023094052007442304, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01886730285714286, "avg_duration_per_task_ms": 206247.68, "avg_output_tokens_per_task": 4550.928571428572, "last_run_timestamp": "2026-10-10T11:50:34.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7398129458917835, "accuracy_std_dev": 0.024891144232638256, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.011375460921843689, "avg_duration_per_task_ms": 475329.3386773547, "avg_output_tokens_per_task": 14172.182364729459, "last_run_timestamp": "2026-10-10T11:52:20.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7396491578947368, "accuracy_std_dev": 0.01866585079556883, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.47321189052631585, "avg_duration_per_task_ms": 648402.0189473684, "avg_output_tokens_per_task": 9813.902105263158, "last_run_timestamp": "2026-10-10T11:08:03.918Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.739166625, "accuracy_std_dev": 0.031523896360291105, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5017313675, "avg_duration_per_task_ms": 190760.71, "avg_output_tokens_per_task": 9915.5125, "last_run_timestamp": "2026-10-10T11:44:47.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7391303686200378, "accuracy_std_dev": 0.02855825218538939, "run_count": 11, "total_questions": 529, "avg_cost_per_task_usd": 0.020223423440453688, "avg_duration_per_task_ms": 99917.1190926276, "avg_output_tokens_per_task": 6754.173913043478, "last_run_timestamp": "2026-10-10T18:46:12.945Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7390713237704918, "accuracy_std_dev": 0.02181217991306695, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.12843605327868854, "avg_duration_per_task_ms": 201024.64344262294, "avg_output_tokens_per_task": 10510.745901639344, "last_run_timestamp": "2026-10-10T19:21:45.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7390475714285715, "accuracy_std_dev": 0.026290010003025565, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.43027001428571426, "avg_duration_per_task_ms": 716198.2485714286, "avg_output_tokens_per_task": 9746.097142857143, "last_run_timestamp": "2026-10-10T11:29:35.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7390474285714286, "accuracy_std_dev": 0.01965692829409479, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014954228571428571, "avg_duration_per_task_ms": 79171.07714285715, "avg_output_tokens_per_task": 4448.311428571428, "last_run_timestamp": "2026-10-10T11:22:45.589Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7388888333333333, "accuracy_std_dev": 0.01383327086431679, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.35860266999999996, "avg_duration_per_task_ms": 86556.65666666666, "avg_output_tokens_per_task": 7118.746666666667, "last_run_timestamp": "2026-10-10T11:09:18.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7388888333333333, "accuracy_std_dev": 0.0319528259544007, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02934656666666667, "avg_duration_per_task_ms": 451531.085, "avg_output_tokens_per_task": 14709.348333333333, "last_run_timestamp": "2026-10-10T13:27:05.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.017992647486688756, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3103866155555556, "avg_duration_per_task_ms": 191198.25333333333, "avg_output_tokens_per_task": 10879.702222222222, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7384840804020101, "accuracy_std_dev": 0.03165357517370423, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.10895770226130652, "avg_duration_per_task_ms": 110251.2148241206, "avg_output_tokens_per_task": 8058.163316582914, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.73833325, "accuracy_std_dev": 0.027436952982018623, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.35042460250000007, "avg_duration_per_task_ms": 221570.3825, "avg_output_tokens_per_task": 10876.18, "last_run_timestamp": "2026-10-10T11:29:08.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7382865983935742, "accuracy_std_dev": 0.027461257286584682, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.07498479919678715, "avg_duration_per_task_ms": 91080.29718875502, "avg_output_tokens_per_task": 4367.955823293173, "last_run_timestamp": "2026-09-26T08:11:26.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7377779166666666, "accuracy_std_dev": 0.037940818112903, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.009713255, "avg_duration_per_task_ms": 395149.92, "avg_output_tokens_per_task": 14577.848333333333, "last_run_timestamp": "2026-10-10T14:25:02.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7377778333333334, "accuracy_std_dev": 0.02484511527723086, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06635720666666667, "avg_duration_per_task_ms": 282495.5133333333, "avg_output_tokens_per_task": 8158.293333333333, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7373332, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.028322468, "avg_duration_per_task_ms": 157393.044, "avg_output_tokens_per_task": 16186.18, "last_run_timestamp": "2026-10-10T11:38:15.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7372221666666667, "accuracy_std_dev": 0.02588554461095142, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02299656, "avg_duration_per_task_ms": 397547.93166666664, "avg_output_tokens_per_task": 12291.753333333334, "last_run_timestamp": "2026-10-10T14:27:59.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7371428571428571, "accuracy_std_dev": 0.038190332503054716, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.34757304285714286, "avg_duration_per_task_ms": 395834.59714285715, "avg_output_tokens_per_task": 9470.625714285714, "last_run_timestamp": "2026-10-10T11:27:02.343Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7369697272727272, "accuracy_std_dev": 0.10754797319428362, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.14869326363636362, "avg_duration_per_task_ms": 623801.8454545455, "avg_output_tokens_per_task": 27614.49090909091, "last_run_timestamp": "2026-10-10T11:09:25.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7368726405570062, "accuracy_std_dev": 0.02640283262471465, "run_count": 23, "total_questions": 1149, "avg_cost_per_task_usd": 0.12460780939947784, "avg_duration_per_task_ms": 171402.6083550914, "avg_output_tokens_per_task": 9190.991296779808, "last_run_timestamp": "2026-10-10T11:58:34.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2879983000000002, "avg_duration_per_task_ms": 310547.73, "avg_output_tokens_per_task": 22913.4, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7366668333333333, "accuracy_std_dev": 0.033054360390205026, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.016296663333333333, "avg_duration_per_task_ms": 149274.76666666666, "avg_output_tokens_per_task": 15030.68, "last_run_timestamp": "2026-10-10T11:18:50.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7366668, "accuracy_std_dev": 0.024221175309220654, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04493056000000001, "avg_duration_per_task_ms": 138671.204, "avg_output_tokens_per_task": 28727.112, "last_run_timestamp": "2026-10-10T11:07:22.568Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7366668, "accuracy_std_dev": 0.022900761755015906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.063241744, "avg_duration_per_task_ms": 1530537.112, "avg_output_tokens_per_task": 25222.716, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7366667499999999, "accuracy_std_dev": 0.029814193116492343, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.049640665, "avg_duration_per_task_ms": 368265.4225, "avg_output_tokens_per_task": 10154.2325, "last_run_timestamp": "2026-10-10T20:13:30.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7366665833333333, "accuracy_std_dev": 0.04004635046097278, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09680749833333332, "avg_duration_per_task_ms": 150044.80833333332, "avg_output_tokens_per_task": 9307.375, "last_run_timestamp": "2026-10-10T11:40:53.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7362964444444444, "accuracy_std_dev": 0.017808576717170822, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.38587963333333336, "avg_duration_per_task_ms": 178574.04, "avg_output_tokens_per_task": 10909.295555555556, "last_run_timestamp": "2026-10-10T11:25:32.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7362542886597938, "accuracy_std_dev": 0.030839009988703172, "run_count": 12, "total_questions": 582, "avg_cost_per_task_usd": 0.058931000000000004, "avg_duration_per_task_ms": 124849.64604810996, "avg_output_tokens_per_task": 7792.945017182131, "last_run_timestamp": "2026-10-10T11:39:50.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.73625, "accuracy_std_dev": 0.03453823565629835, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0753320925, "avg_duration_per_task_ms": 225286.185, "avg_output_tokens_per_task": 11323.5925, "last_run_timestamp": "2026-10-10T18:54:49.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.736, "accuracy_std_dev": 0.014360191892868317, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021830644, "avg_duration_per_task_ms": 202173.696, "avg_output_tokens_per_task": 8712.768, "last_run_timestamp": "2026-10-10T18:44:32.341Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.736, "accuracy_std_dev": 0.03542738426133094, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.42442627200000005, "avg_duration_per_task_ms": 39355.976, "avg_output_tokens_per_task": 9202.272, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7356322298850575, "accuracy_std_dev": 0.05623144531563914, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.29622710057471263, "avg_duration_per_task_ms": 387662.2270114943, "avg_output_tokens_per_task": 10990.514367816091, "last_run_timestamp": "2026-10-10T11:22:20.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13620342666666665, "avg_duration_per_task_ms": 101042.81333333334, "avg_output_tokens_per_task": 8769.966666666667, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7355554166666667, "accuracy_std_dev": 0.0214879236838522, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.007911693333333334, "avg_duration_per_task_ms": 368979.4766666667, "avg_output_tokens_per_task": 14227.998333333333, "last_run_timestamp": "2026-10-10T13:14:11.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7354429620253164, "accuracy_std_dev": 0.021701362929548943, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.020455908860759497, "avg_duration_per_task_ms": 288283.3063291139, "avg_output_tokens_per_task": 10845.830379746836, "last_run_timestamp": "2026-10-10T11:51:18.823Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7354165, "accuracy_std_dev": 0.0429125354005097, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.006874592499999999, "avg_duration_per_task_ms": 421319.53, "avg_output_tokens_per_task": 14336.97, "last_run_timestamp": "2026-10-10T15:31:23.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.735, "accuracy_std_dev": 0.031578860722008335, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05864864000000001, "avg_duration_per_task_ms": 688996.4525, "avg_output_tokens_per_task": 14948.475, "last_run_timestamp": "2026-10-10T19:35:42.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.735, "accuracy_std_dev": 0.011902170673872925, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07856429000000001, "avg_duration_per_task_ms": 285372.9925, "avg_output_tokens_per_task": 13300.165, "last_run_timestamp": "2026-10-10T19:11:19.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.73499995, "accuracy_std_dev": 0.024843857950155427, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.36867235800000003, "avg_duration_per_task_ms": 230064.043, "avg_output_tokens_per_task": 10896.078, "last_run_timestamp": "2026-10-10T11:35:23.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7349998750000001, "accuracy_std_dev": 0.029767504188449777, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1267184125, "avg_duration_per_task_ms": 254157.8425, "avg_output_tokens_per_task": 9997.145, "last_run_timestamp": "2026-10-10T20:45:18.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7349997500000001, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12295462999999998, "avg_duration_per_task_ms": 89986.35, "avg_output_tokens_per_task": 23350.75, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7349397469879518, "accuracy_std_dev": 0.035884660088817896, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.08050074096385541, "avg_duration_per_task_ms": 752519.5602409638, "avg_output_tokens_per_task": 14439.815261044178, "last_run_timestamp": "2026-10-10T21:54:37.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7344444166666667, "accuracy_std_dev": 0.024770657922423504, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023368726666666666, "avg_duration_per_task_ms": 89065.79833333334, "avg_output_tokens_per_task": 7629.631666666667, "last_run_timestamp": "2026-10-10T18:55:15.771Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7344444166666667, "accuracy_std_dev": 0.025507201840716597, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07600889333333334, "avg_duration_per_task_ms": 184831.82166666666, "avg_output_tokens_per_task": 8645.84, "last_run_timestamp": "2026-10-10T11:53:30.354Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7344443333333334, "accuracy_std_dev": 0.023543870268548078, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3636283366666667, "avg_duration_per_task_ms": 194252.73333333334, "avg_output_tokens_per_task": 8764.83, "last_run_timestamp": "2026-10-10T11:18:57.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7342857142857143, "accuracy_std_dev": 0.05044910324479597, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.17247626285714285, "avg_duration_per_task_ms": 282624.91142857145, "avg_output_tokens_per_task": 9879.474285714286, "last_run_timestamp": "2026-10-10T11:32:26.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7340741111111111, "accuracy_std_dev": 0.04903445388804455, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.00916024, "avg_duration_per_task_ms": 334894.9577777778, "avg_output_tokens_per_task": 15526.06, "last_run_timestamp": "2026-10-03T08:11:53.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7338218205128205, "accuracy_std_dev": 0.06370553922722423, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.15103331501831502, "avg_duration_per_task_ms": 1333108.2106227106, "avg_output_tokens_per_task": 27702.578754578753, "last_run_timestamp": "2026-10-10T21:21:31.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7334079284116332, "accuracy_std_dev": 0.030021614385655343, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.07689493288590601, "avg_duration_per_task_ms": 215056.84116331095, "avg_output_tokens_per_task": 8337.460850111856, "last_run_timestamp": "2026-10-10T11:38:34.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7333336666666668, "accuracy_std_dev": 0.03771236166328251, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.44470873333333333, "avg_duration_per_task_ms": 130557.96, "avg_output_tokens_per_task": 20380.893333333333, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.7333335714285714, "accuracy_std_dev": 0.023904731551109535, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019937122857142855, "avg_duration_per_task_ms": 479765.5571428572, "avg_output_tokens_per_task": 14687.234285714285, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7333335, "accuracy_std_dev": 0.033333500000000016, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.009175609999999999, "avg_duration_per_task_ms": 106003.625, "avg_output_tokens_per_task": 5053.315, "last_run_timestamp": "2026-10-10T11:59:02.396Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05192585882668569, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.12086406999999999, "avg_duration_per_task_ms": 154571.83, "avg_output_tokens_per_task": 9530.663333333334, "last_run_timestamp": "2026-10-10T12:38:40.745Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05760637824832321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04898614, "avg_duration_per_task_ms": 106829.63333333333, "avg_output_tokens_per_task": 9897.326666666666, "last_run_timestamp": "2026-10-10T11:51:48.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.73333325, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17039939499999998, "avg_duration_per_task_ms": 114141.445, "avg_output_tokens_per_task": 5083.04, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02503509736024079, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06745117142857142, "avg_duration_per_task_ms": 76562.92857142857, "avg_output_tokens_per_task": 8496.594285714285, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7326620894854586, "accuracy_std_dev": 0.026574598734064515, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.36663018791946306, "avg_duration_per_task_ms": 400926.4272930649, "avg_output_tokens_per_task": 9409.45413870246, "last_run_timestamp": "2026-10-10T11:29:27.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7322223333333333, "accuracy_std_dev": 0.035468537615125265, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.022583826666666664, "avg_duration_per_task_ms": 117756.11, "avg_output_tokens_per_task": 7162.1, "last_run_timestamp": "2026-10-10T19:14:49.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7319933969849246, "accuracy_std_dev": 0.06840954592516678, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.07448894472361808, "avg_duration_per_task_ms": 91970.93467336684, "avg_output_tokens_per_task": 13988.37688442211, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73166675, "accuracy_std_dev": 0.015184321229067165, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.29879598, "avg_duration_per_task_ms": 177541.565, "avg_output_tokens_per_task": 10870.785, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7315942608695652, "accuracy_std_dev": 0.033605750226408763, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.35594851652173914, "avg_duration_per_task_ms": 206193.49739130435, "avg_output_tokens_per_task": 10313.84, "last_run_timestamp": "2026-10-10T11:22:45.965Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.731293, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.12389479591836736, "avg_duration_per_task_ms": 269993.51020408166, "avg_output_tokens_per_task": 23771, "last_run_timestamp": "2026-07-18T08:00:15.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7311113333333332, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10536965333333334, "avg_duration_per_task_ms": 88766.42666666667, "avg_output_tokens_per_task": 4017.0533333333333, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.022662199996175744, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3988358066666667, "avg_duration_per_task_ms": 172929.82666666666, "avg_output_tokens_per_task": 27148.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.16557658, "avg_duration_per_task_ms": 484024.5333333333, "avg_output_tokens_per_task": 29783.94, "last_run_timestamp": "2026-10-10T19:22:43.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7302118929765886, "accuracy_std_dev": 0.012689587626825722, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.2335497525083612, "avg_duration_per_task_ms": 376307.7892976589, "avg_output_tokens_per_task": 9829.47491638796, "last_run_timestamp": "2026-10-10T11:21:19.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7301412182628062, "accuracy_std_dev": 0.02931592753958457, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.06766333184855233, "avg_duration_per_task_ms": 212965.04008908686, "avg_output_tokens_per_task": 11590.710467706014, "last_run_timestamp": "2026-10-10T19:18:15.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.730000125, "accuracy_std_dev": 0.04988867885461965, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07825786500000001, "avg_duration_per_task_ms": 212986.7025, "avg_output_tokens_per_task": 8293.9625, "last_run_timestamp": "2026-09-19T08:12:24.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7299998000000001, "accuracy_std_dev": 0.02290067442150996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045081952, "avg_duration_per_task_ms": 177241.02, "avg_output_tokens_per_task": 31491.432, "last_run_timestamp": "2026-10-10T11:06:58.135Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7296296666666666, "accuracy_std_dev": 0.012216782182264232, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.25703444444444445, "avg_duration_per_task_ms": 284813.7555555556, "avg_output_tokens_per_task": 10375.755555555555, "last_run_timestamp": "2026-10-10T11:00:29.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.728889, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06727646666666667, "avg_duration_per_task_ms": 501138.4733333333, "avg_output_tokens_per_task": 9493.433333333332, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.728889, "accuracy_std_dev": 0.022443443323657393, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05784399555555556, "avg_duration_per_task_ms": 318648.68, "avg_output_tokens_per_task": 9856.32, "last_run_timestamp": "2026-10-10T20:00:56.451Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7288888333333333, "accuracy_std_dev": 0.025867535305969056, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07605682166666666, "avg_duration_per_task_ms": 645935.9233333333, "avg_output_tokens_per_task": 8649.321666666667, "last_run_timestamp": "2026-10-10T11:19:35.083Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7288603933823529, "accuracy_std_dev": 0.04323775063491326, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.021905998161764705, "avg_duration_per_task_ms": 100677.26838235294, "avg_output_tokens_per_task": 7100.75, "last_run_timestamp": "2026-10-10T19:08:09.142Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.728, "accuracy_std_dev": 0.027776952878240627, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04150964, "avg_duration_per_task_ms": 140081.308, "avg_output_tokens_per_task": 18264.704, "last_run_timestamp": "2026-10-10T11:28:56.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7279999, "accuracy_std_dev": 0.03448671628453485, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.30246312399999997, "avg_duration_per_task_ms": 332768.642, "avg_output_tokens_per_task": 9378.478, "last_run_timestamp": "2026-10-10T11:50:42.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7278787272727273, "accuracy_std_dev": 0.027970785626470766, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02821320545454546, "avg_duration_per_task_ms": 208497.22363636363, "avg_output_tokens_per_task": 8992.374545454546, "last_run_timestamp": "2026-10-03T16:24:32.945Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7278566673427991, "accuracy_std_dev": 0.03572243647191496, "run_count": 10, "total_questions": 493, "avg_cost_per_task_usd": 0.10080718052738337, "avg_duration_per_task_ms": 324790.75456389453, "avg_output_tokens_per_task": 14787.588235294117, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.72777775, "accuracy_std_dev": 0.027057441185759493, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07957451833333333, "avg_duration_per_task_ms": 222728.50333333333, "avg_output_tokens_per_task": 8680.433333333332, "last_run_timestamp": "2026-10-10T12:28:42.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7274997500000001, "accuracy_std_dev": 0.035502329736053946, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.064519175, "avg_duration_per_task_ms": 370870.4875, "avg_output_tokens_per_task": 11564.955, "last_run_timestamp": "2026-10-10T19:34:22.239Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7274305937499999, "accuracy_std_dev": 0.02334167860109178, "run_count": 6, "total_questions": 288, "avg_cost_per_task_usd": 0.14647149305555557, "avg_duration_per_task_ms": 714189.5451388889, "avg_output_tokens_per_task": 29008.288194444445, "last_run_timestamp": "2026-10-10T18:53:41.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7273333, "accuracy_std_dev": 0.03495384321659065, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.33412321800000006, "avg_duration_per_task_ms": 205827.788, "avg_output_tokens_per_task": 10339.92, "last_run_timestamp": "2026-10-10T11:19:03.850Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7273159627507164, "accuracy_std_dev": 0.03210912735135148, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.054678793696275076, "avg_duration_per_task_ms": 920429.4957020057, "avg_output_tokens_per_task": 14154.799426934098, "last_run_timestamp": "2026-10-10T19:33:17.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7272728080808081, "accuracy_std_dev": 0.03401092910650194, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12744196464646465, "avg_duration_per_task_ms": 652406.3232323233, "avg_output_tokens_per_task": 27175.661616161615, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7269664179775281, "accuracy_std_dev": 0.05285482315214426, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.061720067415730336, "avg_duration_per_task_ms": 941730.6539325842, "avg_output_tokens_per_task": 65555.35730337078, "last_run_timestamp": "2026-10-10T19:11:36.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11300302000000001, "avg_duration_per_task_ms": 353321.78, "avg_output_tokens_per_task": 3658.14, "last_run_timestamp": "2026-10-03T00:00:06.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.726666625, "accuracy_std_dev": 0.045582550268544374, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0456602475, "avg_duration_per_task_ms": 292041.375, "avg_output_tokens_per_task": 14472.8075, "last_run_timestamp": "2026-10-10T11:51:12.282Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7266666, "accuracy_std_dev": 0.03155246072559159, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.3544039280000001, "avg_duration_per_task_ms": 237853.414, "avg_output_tokens_per_task": 10864.908, "last_run_timestamp": "2026-10-10T11:43:14.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.7266666, "accuracy_std_dev": 0.020221023165013202, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.044841616, "avg_duration_per_task_ms": 129167.684, "avg_output_tokens_per_task": 26225.984, "last_run_timestamp": "2026-10-10T11:48:30.799Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7266665714285715, "accuracy_std_dev": 0.035989231194333456, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11753475142857144, "avg_duration_per_task_ms": 678895.0285714286, "avg_output_tokens_per_task": 31918.602857142858, "last_run_timestamp": "2026-10-10T11:08:14.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7266665, "accuracy_std_dev": 0.03162256578378802, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.325251515, "avg_duration_per_task_ms": 113433.705, "avg_output_tokens_per_task": 10925.445, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7261110000000001, "accuracy_std_dev": 0.026134816662835056, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.013091720000000001, "avg_duration_per_task_ms": 286219.44333333336, "avg_output_tokens_per_task": 19785.066666666666, "last_run_timestamp": "2026-10-10T11:40:53.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7260604545454545, "accuracy_std_dev": 0.023860693807498716, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07136424727272728, "avg_duration_per_task_ms": 198968.38727272727, "avg_output_tokens_per_task": 8450.036363636364, "last_run_timestamp": "2026-10-10T12:55:32.994Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.725714, "accuracy_std_dev": 0.027004721179823374, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04600672857142857, "avg_duration_per_task_ms": 172219.76, "avg_output_tokens_per_task": 8290.077142857142, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7255554166666667, "accuracy_std_dev": 0.021227793045354233, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07936439, "avg_duration_per_task_ms": 253012.47333333333, "avg_output_tokens_per_task": 9156.243333333334, "last_run_timestamp": "2026-10-10T11:53:49.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7254545454545455, "accuracy_std_dev": 0.02738706202293217, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.038856060000000005, "avg_duration_per_task_ms": 146486.05272727273, "avg_output_tokens_per_task": 8570.81090909091, "last_run_timestamp": "2026-10-10T11:39:07.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7251851111111112, "accuracy_std_dev": 0.030107077312834407, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.29754660666666666, "avg_duration_per_task_ms": 378777.12666666665, "avg_output_tokens_per_task": 9437.744444444445, "last_run_timestamp": "2026-10-10T11:49:11.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7251207652173913, "accuracy_std_dev": 0.03173202093295975, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.020070034782608694, "avg_duration_per_task_ms": 306185.0115942029, "avg_output_tokens_per_task": 7045.43768115942, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.7247618571428571, "accuracy_std_dev": 0.02776646862894973, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05254494857142857, "avg_duration_per_task_ms": 272263.16, "avg_output_tokens_per_task": 8601.185714285713, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7247618571428571, "accuracy_std_dev": 0.03161563234689254, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034889062857142855, "avg_duration_per_task_ms": 356175.16285714286, "avg_output_tokens_per_task": 8342.902857142857, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7245833125, "accuracy_std_dev": 0.036187558051281155, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.06812713125000001, "avg_duration_per_task_ms": 121769.8325, "avg_output_tokens_per_task": 8642.055, "last_run_timestamp": "2026-10-10T12:45:34.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.7244897346938776, "accuracy_std_dev": 0.009027500000000022, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.08190116326530611, "avg_duration_per_task_ms": 1045226.0918367347, "avg_output_tokens_per_task": 15985.285714285714, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.7244446666666666, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05245255333333333, "avg_duration_per_task_ms": 276706.37333333335, "avg_output_tokens_per_task": 8448.833333333334, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7244445555555555, "accuracy_std_dev": 0.025337407080840913, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.054274675555555554, "avg_duration_per_task_ms": 301393.08222222223, "avg_output_tokens_per_task": 11200.555555555555, "last_run_timestamp": "2026-10-10T18:58:00.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7244444444444444, "accuracy_std_dev": 0.017497922277364575, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10756234222222222, "avg_duration_per_task_ms": 333076.56666666665, "avg_output_tokens_per_task": 14814.204444444444, "last_run_timestamp": "2026-10-10T19:04:05.899Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7240565707547171, "accuracy_std_dev": 0.03163321895945818, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.11481138679245283, "avg_duration_per_task_ms": 107730.78773584905, "avg_output_tokens_per_task": 9300.52358490566, "last_run_timestamp": "2026-10-10T11:50:06.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7238493974895398, "accuracy_std_dev": 0.03039350529715622, "run_count": 24, "total_questions": 1195, "avg_cost_per_task_usd": 0.11536971046025105, "avg_duration_per_task_ms": 139092.2150627615, "avg_output_tokens_per_task": 8422.310460251047, "last_run_timestamp": "2026-10-10T12:27:41.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7238442007299271, "accuracy_std_dev": 0.026621319896989543, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.04390420437956204, "avg_duration_per_task_ms": 215259.2098540146, "avg_output_tokens_per_task": 9833.600364963504, "last_run_timestamp": "2026-10-10T12:00:06.272Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7233333333333334, "accuracy_std_dev": 0.023959997528287768, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06615269666666666, "avg_duration_per_task_ms": 207301.43333333332, "avg_output_tokens_per_task": 9384.513333333334, "last_run_timestamp": "2026-10-10T11:22:15.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.72333325, "accuracy_std_dev": 0.01374358436462263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011429815, "avg_duration_per_task_ms": 101515.345, "avg_output_tokens_per_task": 5183.985, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7230575714285715, "accuracy_std_dev": 0.02915508369237475, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.13198787969924813, "avg_duration_per_task_ms": 71794.78947368421, "avg_output_tokens_per_task": 10252.233082706767, "last_run_timestamp": "2026-10-10T19:19:07.184Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7230303636363636, "accuracy_std_dev": 0.04122986471936161, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12495049272727274, "avg_duration_per_task_ms": 180996.27272727274, "avg_output_tokens_per_task": 12125.23090909091, "last_run_timestamp": "2026-10-10T12:09:55.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.722963, "accuracy_std_dev": 0.01808372541510429, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.051240302222222214, "avg_duration_per_task_ms": 386538.15555555554, "avg_output_tokens_per_task": 9392.624444444444, "last_run_timestamp": "2026-10-10T20:11:19.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.7228571428571429, "accuracy_std_dev": 0.026393263417711994, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3014312628571429, "avg_duration_per_task_ms": 206224.79142857142, "avg_output_tokens_per_task": 11626.885714285714, "last_run_timestamp": "2026-10-10T12:42:09.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7226666, "accuracy_std_dev": 0.01959597237801686, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.033446764, "avg_duration_per_task_ms": 104528.44, "avg_output_tokens_per_task": 23630.596, "last_run_timestamp": "2026-10-10T11:30:58.693Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7222221428571428, "accuracy_std_dev": 0.024312166707227047, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.046834047619047615, "avg_duration_per_task_ms": 58108.6462585034, "avg_output_tokens_per_task": 13243.625850340137, "last_run_timestamp": "2026-09-29T21:00:05.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7222221111111111, "accuracy_std_dev": 0.027917975508621698, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.13745050505050505, "avg_duration_per_task_ms": 712939.898989899, "avg_output_tokens_per_task": 27938.80808080808, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.722222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019033145833333334, "avg_duration_per_task_ms": 65993.54166666667, "avg_output_tokens_per_task": 3355.9791666666665, "last_run_timestamp": "2026-09-12T08:12:22.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7219429246231156, "accuracy_std_dev": 0.09608921942840087, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.07445455527638191, "avg_duration_per_task_ms": 193168.42211055275, "avg_output_tokens_per_task": 23981.597989949747, "last_run_timestamp": "2026-10-10T18:47:18.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.7216665, "accuracy_std_dev": 0.011666500000000024, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.36281153000000005, "avg_duration_per_task_ms": 114354.7, "avg_output_tokens_per_task": 9018.87, "last_run_timestamp": "2026-10-10T11:34:33.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7211113333333333, "accuracy_std_dev": 0.015595100530472852, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017302566666666668, "avg_duration_per_task_ms": 811343.51, "avg_output_tokens_per_task": 20213.606666666667, "last_run_timestamp": "2026-10-10T11:19:51.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7211000606060606, "accuracy_std_dev": 0.07754356597114187, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.04577266666666666, "avg_duration_per_task_ms": 266350.4158249158, "avg_output_tokens_per_task": 25957.685185185186, "last_run_timestamp": "2026-10-10T19:43:28.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7210078506375227, "accuracy_std_dev": 0.03602466228677285, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.10133114936247721, "avg_duration_per_task_ms": 207370.83424408015, "avg_output_tokens_per_task": 10427.30965391621, "last_run_timestamp": "2026-10-10T12:12:49.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.72083325, "accuracy_std_dev": 0.03748153063760203, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06457228749999999, "avg_duration_per_task_ms": 75803.1425, "avg_output_tokens_per_task": 9417.155, "last_run_timestamp": "2026-10-10T11:05:58.282Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7207406666666667, "accuracy_std_dev": 0.024634474601437903, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06025161999999999, "avg_duration_per_task_ms": 307034.87555555557, "avg_output_tokens_per_task": 9651.193333333333, "last_run_timestamp": "2026-10-10T19:54:45.316Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7205555833333334, "accuracy_std_dev": 0.037164138347019274, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.046787416666666665, "avg_duration_per_task_ms": 151030.90833333333, "avg_output_tokens_per_task": 8556.955, "last_run_timestamp": "2026-10-10T11:51:25.350Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7205384141414142, "accuracy_std_dev": 0.051112796809483045, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 1.6761988181818182, "avg_duration_per_task_ms": 313695.66161616164, "avg_output_tokens_per_task": 23404.97474747475, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03553636, "avg_duration_per_task_ms": 263015.66, "avg_output_tokens_per_task": 14384.46, "last_run_timestamp": "2026-07-18T16:00:09.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.72, "accuracy_std_dev": 0.03231762505197435, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.36211610499999997, "avg_duration_per_task_ms": 150104.23, "avg_output_tokens_per_task": 23123.4, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7199996666666667, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09322211333333333, "avg_duration_per_task_ms": 149908.11333333334, "avg_output_tokens_per_task": 49346.81333333333, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7193332, "accuracy_std_dev": 0.03143937799893632, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045152332, "avg_duration_per_task_ms": 326196.928, "avg_output_tokens_per_task": 13427.836, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7192392617449664, "accuracy_std_dev": 0.027973240079682514, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.06318058053691275, "avg_duration_per_task_ms": 493682.4899328859, "avg_output_tokens_per_task": 14513.312080536913, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7188551919191919, "accuracy_std_dev": 0.027525589315898408, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.29904142424242425, "avg_duration_per_task_ms": 154139.7323232323, "avg_output_tokens_per_task": 5036.570707070707, "last_run_timestamp": "2026-10-01T19:05:12.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7184344343434343, "accuracy_std_dev": 0.09707188521882894, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.07242429797979798, "avg_duration_per_task_ms": 168379.42676767678, "avg_output_tokens_per_task": 14277.45202020202, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7180952857142857, "accuracy_std_dev": 0.033752526067220005, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.13201746285714283, "avg_duration_per_task_ms": 330154.5142857143, "avg_output_tokens_per_task": 10313.082857142857, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7175758181818181, "accuracy_std_dev": 0.03143352307451446, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009597009090909091, "avg_duration_per_task_ms": 350866.65454545454, "avg_output_tokens_per_task": 14254.325454545455, "last_run_timestamp": "2026-10-10T13:51:45.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7172549411764706, "accuracy_std_dev": 0.014106710908060687, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.037945944705882355, "avg_duration_per_task_ms": 187615.06352941177, "avg_output_tokens_per_task": 9902.724705882352, "last_run_timestamp": "2026-10-10T19:20:40.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7170904591700134, "accuracy_std_dev": 0.030739071735857975, "run_count": 15, "total_questions": 747, "avg_cost_per_task_usd": 0.437738623828648, "avg_duration_per_task_ms": 237840.68808567603, "avg_output_tokens_per_task": 10036.461847389559, "last_run_timestamp": "2026-10-10T11:37:21.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.7170371111111111, "accuracy_std_dev": 0.016956299841680886, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12728777111111111, "avg_duration_per_task_ms": 101323.44222222222, "avg_output_tokens_per_task": 3885.56, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.717037, "accuracy_std_dev": 0.018083848299887195, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.025146933333333333, "avg_duration_per_task_ms": 216796.8177777778, "avg_output_tokens_per_task": 10040.375555555556, "last_run_timestamp": "2026-10-10T19:16:26.718Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015077799999999999, "avg_duration_per_task_ms": 476878.72, "avg_output_tokens_per_task": 26097.54, "last_run_timestamp": "2026-09-28T09:00:02.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7161110833333333, "accuracy_std_dev": 0.034151986458717, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03611448666666667, "avg_duration_per_task_ms": 183171.97833333333, "avg_output_tokens_per_task": 9167.023333333333, "last_run_timestamp": "2026-10-10T12:11:35.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7157575454545455, "accuracy_std_dev": 0.03731083147233555, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1024602309090909, "avg_duration_per_task_ms": 114222.2890909091, "avg_output_tokens_per_task": 9174.52, "last_run_timestamp": "2026-10-10T11:10:05.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7155558333333333, "accuracy_std_dev": 0.0289741860421575, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04852732, "avg_duration_per_task_ms": 969869.3766666667, "avg_output_tokens_per_task": 8596.123333333333, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.040855070557058414, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012743326666666667, "avg_duration_per_task_ms": 1224143.2066666668, "avg_output_tokens_per_task": 18669.653333333332, "last_run_timestamp": "2026-08-29T09:20:05.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7155553333333334, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13937678, "avg_duration_per_task_ms": 263819.3466666667, "avg_output_tokens_per_task": 10311.9, "last_run_timestamp": "2026-10-03T16:18:50.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.715488212121212, "accuracy_std_dev": 0.013403360380889571, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04391666666666666, "avg_duration_per_task_ms": 58944.35858585859, "avg_output_tokens_per_task": 8110.530303030303, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.009582875, "avg_duration_per_task_ms": 164124.5, "avg_output_tokens_per_task": 18445.333333333332, "last_run_timestamp": "2026-10-10T11:44:02.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.715, "accuracy_std_dev": 0.029202378139459823, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09407665500000001, "avg_duration_per_task_ms": 83159.84, "avg_output_tokens_per_task": 3660.855, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7148361502890174, "accuracy_std_dev": 0.08759138393281216, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00943905780346821, "avg_duration_per_task_ms": 208132.0809248555, "avg_output_tokens_per_task": 4098.601156069364, "last_run_timestamp": "2026-10-10T11:51:58.383Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7148146666666667, "accuracy_std_dev": 0.031075936024161484, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09082762666666666, "avg_duration_per_task_ms": 156907.88888888888, "avg_output_tokens_per_task": 48272.70222222222, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7142424545454545, "accuracy_std_dev": 0.0319117780689303, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1133730290909091, "avg_duration_per_task_ms": 98039.63272727272, "avg_output_tokens_per_task": 10090.498181818182, "last_run_timestamp": "2026-10-10T11:41:14.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.714166625, "accuracy_std_dev": 0.0311691275781722, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0320175475, "avg_duration_per_task_ms": 366948.7475, "avg_output_tokens_per_task": 5303.03, "last_run_timestamp": "2026-10-03T08:46:44.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.7140741111111111, "accuracy_std_dev": 0.02141748193231379, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07203324000000001, "avg_duration_per_task_ms": 281427.5711111111, "avg_output_tokens_per_task": 10764.266666666666, "last_run_timestamp": "2026-10-10T19:48:33.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7139392727272728, "accuracy_std_dev": 0.061723041464698515, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.2911906890909091, "avg_duration_per_task_ms": 424451.22363636363, "avg_output_tokens_per_task": 9948.334545454545, "last_run_timestamp": "2026-10-10T11:29:58.824Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7133736491935484, "accuracy_std_dev": 0.10276651148311886, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.2060511633064516, "avg_duration_per_task_ms": 1153803.5564516129, "avg_output_tokens_per_task": 55616.0504032258, "last_run_timestamp": "2026-10-10T19:16:51.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7129095028901734, "accuracy_std_dev": 0.03295967347643888, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.05974209248554914, "avg_duration_per_task_ms": 481297.3583815029, "avg_output_tokens_per_task": 14073.28612716763, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.712121101010101, "accuracy_std_dev": 0.029111223892169177, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12218062626262625, "avg_duration_per_task_ms": 537818.2474747475, "avg_output_tokens_per_task": 55422.06060606061, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.71166675, "accuracy_std_dev": 0.019075733568790994, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07509325, "avg_duration_per_task_ms": 322731.32, "avg_output_tokens_per_task": 33483.435, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7116666666666667, "accuracy_std_dev": 0.03762638455776595, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08100557333333333, "avg_duration_per_task_ms": 79109.17, "avg_output_tokens_per_task": 8517.581666666667, "last_run_timestamp": "2026-10-10T11:36:38.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7114282857142857, "accuracy_std_dev": 0.029323450716420958, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.2380128942857143, "avg_duration_per_task_ms": 294855.26, "avg_output_tokens_per_task": 21043.11142857143, "last_run_timestamp": "2026-10-10T11:21:52.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.7113333333333334, "accuracy_std_dev": 0.03114610968037938, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.13518174266666666, "avg_duration_per_task_ms": 137112.55866666668, "avg_output_tokens_per_task": 8948.754666666666, "last_run_timestamp": "2026-10-10T11:51:03.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.011331263428624743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07186812, "avg_duration_per_task_ms": 348295.26, "avg_output_tokens_per_task": 33945.17333333333, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.7104444, "accuracy_std_dev": 0.028359365645937846, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.12106834666666669, "avg_duration_per_task_ms": 90872.85466666667, "avg_output_tokens_per_task": 10614.206666666667, "last_run_timestamp": "2026-10-10T12:07:01.508Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7103705555555555, "accuracy_std_dev": 0.030809822424065814, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03036840888888889, "avg_duration_per_task_ms": 105409.86666666667, "avg_output_tokens_per_task": 5076.06, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.710000125, "accuracy_std_dev": 0.03144646685812851, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.043149095, "avg_duration_per_task_ms": 372456.3975, "avg_output_tokens_per_task": 9746.7325, "last_run_timestamp": "2026-10-10T20:36:24.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12391315, "avg_duration_per_task_ms": 76197.77, "avg_output_tokens_per_task": 4134.025, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7096297777777777, "accuracy_std_dev": 0.043331717602641394, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06758097555555556, "avg_duration_per_task_ms": 341751.31333333335, "avg_output_tokens_per_task": 10011.704444444444, "last_run_timestamp": "2026-10-10T19:49:16.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7095516447368422, "accuracy_std_dev": 0.03406448599188871, "run_count": 28, "total_questions": 1368, "avg_cost_per_task_usd": 0.09692296418128656, "avg_duration_per_task_ms": 73548.15350877192, "avg_output_tokens_per_task": 8046.236111111111, "last_run_timestamp": "2026-10-10T11:52:20.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7095158747913188, "accuracy_std_dev": 0.03442095606966778, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.07650697662771284, "avg_duration_per_task_ms": 239472.96160267113, "avg_output_tokens_per_task": 8830.707846410685, "last_run_timestamp": "2026-10-10T11:59:03.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.7088887777777778, "accuracy_std_dev": 0.013333370373096707, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.026421224444444447, "avg_duration_per_task_ms": 218382.79777777777, "avg_output_tokens_per_task": 10082.562222222223, "last_run_timestamp": "2026-10-10T18:51:03.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.01749773883169545, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11639628666666667, "avg_duration_per_task_ms": 755791.72, "avg_output_tokens_per_task": 27174.273333333334, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7084847272727273, "accuracy_std_dev": 0.04178733689026097, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09325543272727273, "avg_duration_per_task_ms": 214793.19090909092, "avg_output_tokens_per_task": 8277.278181818181, "last_run_timestamp": "2026-10-10T12:11:48.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08614948, "avg_duration_per_task_ms": 52133.28, "avg_output_tokens_per_task": 7649.3, "last_run_timestamp": "2026-07-18T08:04:46.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.7066665000000001, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.054561849999999995, "avg_duration_per_task_ms": 281726.22, "avg_output_tokens_per_task": 8888.58, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7056452258064517, "accuracy_std_dev": 0.012405393094940622, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.6654760282258064, "avg_duration_per_task_ms": 686538.5927419355, "avg_output_tokens_per_task": 55521.342741935485, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7043380136986301, "accuracy_std_dev": 0.016697582419819515, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.027861123287671232, "avg_duration_per_task_ms": 317404.41780821915, "avg_output_tokens_per_task": 16264.835616438357, "last_run_timestamp": "2026-10-10T20:55:29.743Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7042423636363636, "accuracy_std_dev": 0.1528256864263529, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05846522, "avg_duration_per_task_ms": 152693.06363636363, "avg_output_tokens_per_task": 18465.061818181817, "last_run_timestamp": "2026-10-10T19:04:13.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.028782815862246673, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.059492744, "avg_duration_per_task_ms": 531413.344, "avg_output_tokens_per_task": 14170.264, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.702381, "accuracy_std_dev": 0.02809942180391412, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1988400742857143, "avg_duration_per_task_ms": 1709336.88, "avg_output_tokens_per_task": 13446.757142857143, "last_run_timestamp": "2026-10-10T18:42:31.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.7016668749999999, "accuracy_std_dev": 0.05052503642363232, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.030146600000000003, "avg_duration_per_task_ms": 298672.2075, "avg_output_tokens_per_task": 10079.875, "last_run_timestamp": "2026-10-10T18:56:27.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.701111, "accuracy_std_dev": 0.011811220865487748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13736028333333333, "avg_duration_per_task_ms": 494538.36333333334, "avg_output_tokens_per_task": 57774.223333333335, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7011108333333333, "accuracy_std_dev": 0.020608011147906124, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.051610086666666666, "avg_duration_per_task_ms": 398260.51, "avg_output_tokens_per_task": 9237.883333333333, "last_run_timestamp": "2026-10-03T16:12:07.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.7008929330357142, "accuracy_std_dev": 0.031656430987076, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.04512097991071429, "avg_duration_per_task_ms": 429480.1316964286, "avg_output_tokens_per_task": 9115.185267857143, "last_run_timestamp": "2026-10-10T18:33:03.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.7000003333333334, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010915493333333333, "avg_duration_per_task_ms": 115952.82, "avg_output_tokens_per_task": 5345.16, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7000003333333332, "accuracy_std_dev": 0.024944382578492925, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12734470666666667, "avg_duration_per_task_ms": 591806.9066666666, "avg_output_tokens_per_task": 57532.753333333334, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.70000025, "accuracy_std_dev": 0.03162288201109283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12919751, "avg_duration_per_task_ms": 506623.765, "avg_output_tokens_per_task": 10892.11, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.019795359999999998, "avg_duration_per_task_ms": 396615.94, "avg_output_tokens_per_task": 23218.96, "last_run_timestamp": "2026-10-01T08:03:06.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01539542, "avg_duration_per_task_ms": 2335452.5, "avg_output_tokens_per_task": 25378.6, "last_run_timestamp": "2026-07-18T08:04:08.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.6998645487804879, "accuracy_std_dev": 0.05531710950158186, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.031559408536585364, "avg_duration_per_task_ms": 1019512.3414634146, "avg_output_tokens_per_task": 43385.81707317073, "last_run_timestamp": "2026-10-03T16:10:32.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.6986666, "accuracy_std_dev": 0.007483243510671025, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04888268, "avg_duration_per_task_ms": 227205.572, "avg_output_tokens_per_task": 6367.684, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6985507420289855, "accuracy_std_dev": 0.027723151851321442, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.01719656231884058, "avg_duration_per_task_ms": 309641.62898550724, "avg_output_tokens_per_task": 9142.672463768116, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6977778333333333, "accuracy_std_dev": 0.026573791972647694, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04297996, "avg_duration_per_task_ms": 184061.30666666667, "avg_output_tokens_per_task": 8458.556666666667, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6977778333333333, "accuracy_std_dev": 0.017915990515148434, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21024033333333336, "avg_duration_per_task_ms": 335029.48333333334, "avg_output_tokens_per_task": 34166.46666666667, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.6972790204081633, "accuracy_std_dev": 0.04361100000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.08154698979591836, "avg_duration_per_task_ms": 436567.05102040817, "avg_output_tokens_per_task": 13545.377551020409, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.6970371111111111, "accuracy_std_dev": 0.05417024882195113, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10031135333333334, "avg_duration_per_task_ms": 244307.61111111112, "avg_output_tokens_per_task": 10714.342222222222, "last_run_timestamp": "2026-10-10T18:48:20.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.69666675, "accuracy_std_dev": 0.12892723184489574, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04194705, "avg_duration_per_task_ms": 293196.69, "avg_output_tokens_per_task": 10603.27, "last_run_timestamp": "2026-10-10T20:59:28.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14710045000000002, "avg_duration_per_task_ms": 163276.98, "avg_output_tokens_per_task": 5053.91, "last_run_timestamp": "2026-10-01T08:02:59.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.696561865470852, "accuracy_std_dev": 0.012961611737724062, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.021608876681614344, "avg_duration_per_task_ms": 698943.9192825112, "avg_output_tokens_per_task": 19226.64125560538, "last_run_timestamp": "2026-10-10T18:35:46.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.6958332500000001, "accuracy_std_dev": 0.08906522812628674, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0512019375, "avg_duration_per_task_ms": 570157.7625, "avg_output_tokens_per_task": 19195.9675, "last_run_timestamp": "2026-10-10T19:12:44.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6951850555555555, "accuracy_std_dev": 0.023810538468576885, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.10653520444444445, "avg_duration_per_task_ms": 77015.53444444445, "avg_output_tokens_per_task": 9317.83111111111, "last_run_timestamp": "2026-10-10T11:34:42.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6940235606060606, "accuracy_std_dev": 0.024175874322025556, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.034200517676767676, "avg_duration_per_task_ms": 689874.9823232323, "avg_output_tokens_per_task": 19910.09595959596, "last_run_timestamp": "2026-10-10T19:24:00.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.69333325, "accuracy_std_dev": 0.05676452335030659, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.22464821499999998, "avg_duration_per_task_ms": 688281.77, "avg_output_tokens_per_task": 10611.7, "last_run_timestamp": "2026-10-10T14:01:53.590Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.693333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08382193, "avg_duration_per_task_ms": 858486.8, "avg_output_tokens_per_task": 15204.12, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.060288180000000004, "avg_duration_per_task_ms": 350636.38, "avg_output_tokens_per_task": 10540.6, "last_run_timestamp": "2026-07-18T08:04:08.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02187568, "avg_duration_per_task_ms": 236451.76, "avg_output_tokens_per_task": 11760.74, "last_run_timestamp": "2026-07-18T08:04:03.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6929823609022557, "accuracy_std_dev": 0.026094683447675196, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.03234121052631579, "avg_duration_per_task_ms": 241069.54887218046, "avg_output_tokens_per_task": 9648.423558897242, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6924997500000001, "accuracy_std_dev": 0.0419573344802491, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.019822750000000004, "avg_duration_per_task_ms": 438051.1125, "avg_output_tokens_per_task": 20830, "last_run_timestamp": "2026-10-10T20:24:13.976Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6923076113360324, "accuracy_std_dev": 0.004408577938519381, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.009435951417004049, "avg_duration_per_task_ms": 259361.54655870446, "avg_output_tokens_per_task": 9080.562753036438, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.69, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01726069, "avg_duration_per_task_ms": 298145.74, "avg_output_tokens_per_task": 10516.76, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6891892297297297, "accuracy_std_dev": 0.03170486152346707, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6622049662162162, "avg_duration_per_task_ms": 777791.8783783783, "avg_output_tokens_per_task": 59355.4527027027, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6888886666666667, "accuracy_std_dev": 0.021998731726069, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015484113333333334, "avg_duration_per_task_ms": 264745.86666666664, "avg_output_tokens_per_task": 18035.506666666668, "last_run_timestamp": "2026-10-10T20:39:08.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.6887609855907781, "accuracy_std_dev": 0.02815688345390403, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.019089083573487032, "avg_duration_per_task_ms": 454304.7060518732, "avg_output_tokens_per_task": 9047.129682997118, "last_run_timestamp": "2026-10-10T19:16:30.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.688485, "accuracy_std_dev": 0.07169463814489388, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11210284363636364, "avg_duration_per_task_ms": 69179.89090909091, "avg_output_tokens_per_task": 8846.072727272727, "last_run_timestamp": "2026-10-10T11:52:26.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6879166250000001, "accuracy_std_dev": 0.07218680699569953, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05169213000000001, "avg_duration_per_task_ms": 304751.5525, "avg_output_tokens_per_task": 8046.7025, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.6877776666666667, "accuracy_std_dev": 0.010304212256267955, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11416634666666667, "avg_duration_per_task_ms": 162037.17333333334, "avg_output_tokens_per_task": 8889.293333333333, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6876191428571429, "accuracy_std_dev": 0.0358884483600765, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.03518511571428571, "avg_duration_per_task_ms": 165038.81714285715, "avg_output_tokens_per_task": 10124.521428571428, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6872909230769231, "accuracy_std_dev": 0.021147758508094125, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.006207953177257525, "avg_duration_per_task_ms": 252164.59866220737, "avg_output_tokens_per_task": 9519.789297658863, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02072933, "avg_duration_per_task_ms": 450665.84, "avg_output_tokens_per_task": 10600.05, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.6855555, "accuracy_std_dev": 0.052973362745660516, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.018873013333333334, "avg_duration_per_task_ms": 269968.75333333336, "avg_output_tokens_per_task": 10405.67, "last_run_timestamp": "2026-10-10T19:21:53.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.07779388246014092, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00909372, "avg_duration_per_task_ms": 253755.2, "avg_output_tokens_per_task": 15941.146666666667, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.68333325, "accuracy_std_dev": 0.06368309125024867, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.05480554, "avg_duration_per_task_ms": 1657352.435, "avg_output_tokens_per_task": 54132.445, "last_run_timestamp": "2026-10-03T08:39:18.140Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.6824557368421051, "accuracy_std_dev": 0.05773499999999998, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.19861939999999997, "avg_duration_per_task_ms": 328265.04210526316, "avg_output_tokens_per_task": 23629.65263157895, "last_run_timestamp": "2026-10-03T16:20:06.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.058584163368230795, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02942621, "avg_duration_per_task_ms": 175208.89333333334, "avg_output_tokens_per_task": 10644.23, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6820807398843931, "accuracy_std_dev": 0.018427811270036426, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.015085965317919076, "avg_duration_per_task_ms": 243924.5809248555, "avg_output_tokens_per_task": 11146.326589595375, "last_run_timestamp": "2026-08-30T00:01:09.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.681992448275862, "accuracy_std_dev": 0.10159207419370662, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07432972701149426, "avg_duration_per_task_ms": 819407.2413793104, "avg_output_tokens_per_task": 46149.24425287356, "last_run_timestamp": "2026-10-10T18:48:14.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6819047142857143, "accuracy_std_dev": 0.025376904886779485, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007673354285714286, "avg_duration_per_task_ms": 259027.58, "avg_output_tokens_per_task": 11768.4, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6816665000000001, "accuracy_std_dev": 0.02327362599059285, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013877145, "avg_duration_per_task_ms": 70751.9, "avg_output_tokens_per_task": 5304.49, "last_run_timestamp": "2026-10-01T19:04:52.652Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.68, "accuracy_std_dev": 0.01440159456449179, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015966806666666666, "avg_duration_per_task_ms": 259752.2, "avg_output_tokens_per_task": 10038.846666666666, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.6777777777777778, "accuracy_std_dev": 0.056655654704103615, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02349592666666667, "avg_duration_per_task_ms": 202633.27333333335, "avg_output_tokens_per_task": 9395.406666666666, "last_run_timestamp": "2026-10-10T19:22:03.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.6777777777777778, "accuracy_std_dev": 0.07894361482135037, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07500947555555555, "avg_duration_per_task_ms": 278493.0911111111, "avg_output_tokens_per_task": 25748.14, "last_run_timestamp": "2026-10-10T18:46:46.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04451592, "avg_duration_per_task_ms": 466424.76, "avg_output_tokens_per_task": 10206.78, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6761905714285714, "accuracy_std_dev": 0.01880810362474601, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009157782857142858, "avg_duration_per_task_ms": 249382.73428571428, "avg_output_tokens_per_task": 10084.637142857142, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.6757575454545455, "accuracy_std_dev": 0.03827318025687446, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.017890945454545452, "avg_duration_per_task_ms": 338670.8781818182, "avg_output_tokens_per_task": 20078.554545454546, "last_run_timestamp": "2026-10-10T21:10:48.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.6741667499999999, "accuracy_std_dev": 0.0918596083103858, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.090455615, "avg_duration_per_task_ms": 88078.0775, "avg_output_tokens_per_task": 8488.9925, "last_run_timestamp": "2026-10-10T18:41:45.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64995528, "avg_duration_per_task_ms": 85231.23, "avg_output_tokens_per_task": 3880.06, "last_run_timestamp": "2026-10-01T08:05:26.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6733333636363636, "accuracy_std_dev": 0.06381806526834732, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06596764181818182, "avg_duration_per_task_ms": 779394.0927272728, "avg_output_tokens_per_task": 18514.336363636365, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.02372680934208296, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11501549333333332, "avg_duration_per_task_ms": 145439.36666666667, "avg_output_tokens_per_task": 56059.84, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6733330000000001, "accuracy_std_dev": 0.016329931618554488, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008951993333333332, "avg_duration_per_task_ms": 824834.48, "avg_output_tokens_per_task": 18340.86, "last_run_timestamp": "2026-10-10T19:34:42.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09000018000000001, "avg_duration_per_task_ms": 1674604.48, "avg_output_tokens_per_task": 17624.46, "last_run_timestamp": "2026-07-18T16:00:08.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.672916875, "accuracy_std_dev": 0.0946110438485348, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.056751722500000004, "avg_duration_per_task_ms": 524628.53, "avg_output_tokens_per_task": 16401.87, "last_run_timestamp": "2026-10-10T19:25:46.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6726727747747747, "accuracy_std_dev": 0.04092304282098771, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.023259461711711713, "avg_duration_per_task_ms": 273637.62387387385, "avg_output_tokens_per_task": 17560.18018018018, "last_run_timestamp": "2026-10-10T19:28:16.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.67, "accuracy_std_dev": 0.017384450270284638, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.017308236, "avg_duration_per_task_ms": 120715.156, "avg_output_tokens_per_task": 21418.432, "last_run_timestamp": "2026-10-10T11:31:03.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6698850000000001, "accuracy_std_dev": 0.030556564488017048, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.12515819931034483, "avg_duration_per_task_ms": 120179.14275862068, "avg_output_tokens_per_task": 12653.900689655173, "last_run_timestamp": "2026-10-10T11:29:00.634Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.6685714285714286, "accuracy_std_dev": 0.01875964453559928, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.029877088571428574, "avg_duration_per_task_ms": 242438.32571428572, "avg_output_tokens_per_task": 11057.357142857143, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6679999, "accuracy_std_dev": 0.03383610563421859, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11777596400000001, "avg_duration_per_task_ms": 124529.096, "avg_output_tokens_per_task": 12738.322, "last_run_timestamp": "2026-10-10T11:22:45.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6663334, "accuracy_std_dev": 0.02651835925995422, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.114373532, "avg_duration_per_task_ms": 125231.99, "avg_output_tokens_per_task": 12239.046, "last_run_timestamp": "2026-10-10T11:19:31.087Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6628353908045977, "accuracy_std_dev": 0.06360316641360503, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.024118077586206897, "avg_duration_per_task_ms": 468405.7327586207, "avg_output_tokens_per_task": 16303.336206896553, "last_run_timestamp": "2026-10-10T19:08:25.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6619998, "accuracy_std_dev": 0.015289659543626185, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.02312782, "avg_duration_per_task_ms": 165165.768, "avg_output_tokens_per_task": 17110.856, "last_run_timestamp": "2026-10-10T18:59:10.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6600001428571428, "accuracy_std_dev": 0.024688651721727035, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02290525142857143, "avg_duration_per_task_ms": 311367.0028571429, "avg_output_tokens_per_task": 10406.782857142856, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00861218, "avg_duration_per_task_ms": 379136.4, "avg_output_tokens_per_task": 24201, "last_run_timestamp": "2026-07-18T16:00:10.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.6588888333333334, "accuracy_std_dev": 0.02354406690595791, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022377286666666666, "avg_duration_per_task_ms": 254720.16666666666, "avg_output_tokens_per_task": 9788.826666666666, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6580951428571429, "accuracy_std_dev": 0.018418057788319512, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.026633637142857144, "avg_duration_per_task_ms": 383567.4857142857, "avg_output_tokens_per_task": 10552.454285714286, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6579147260273973, "accuracy_std_dev": 0.11208961231097506, "run_count": 9, "total_questions": 438, "avg_cost_per_task_usd": 0.36986931735159817, "avg_duration_per_task_ms": 1949962.3150684931, "avg_output_tokens_per_task": 62327.19634703196, "last_run_timestamp": "2026-10-10T11:38:58.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6573333, "accuracy_std_dev": 0.028263311647375403, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.12181550133333331, "avg_duration_per_task_ms": 120760.41533333334, "avg_output_tokens_per_task": 12566.805333333334, "last_run_timestamp": "2026-10-10T11:31:06.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6561111666666667, "accuracy_std_dev": 0.04057057332359941, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.021655356666666667, "avg_duration_per_task_ms": 227541.03, "avg_output_tokens_per_task": 28375, "last_run_timestamp": "2026-10-10T11:34:56.282Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6557823102040816, "accuracy_std_dev": 0.009318408855593308, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.014797057142857142, "avg_duration_per_task_ms": 277907.2775510204, "avg_output_tokens_per_task": 20245.579591836733, "last_run_timestamp": "2026-10-10T11:18:54.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.6540179419642858, "accuracy_std_dev": 0.10628475367719596, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.09064835491071428, "avg_duration_per_task_ms": 220062.35267857142, "avg_output_tokens_per_task": 23673.90401785714, "last_run_timestamp": "2026-10-10T18:54:19.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.023726809342082957, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03210012, "avg_duration_per_task_ms": 143919.89333333334, "avg_output_tokens_per_task": 17353.273333333334, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.65166675, "accuracy_std_dev": 0.012801703448662586, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.031589305, "avg_duration_per_task_ms": 139715.76, "avg_output_tokens_per_task": 17049.6, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.649933124497992, "accuracy_std_dev": 0.01992940230513702, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.01677981124497992, "avg_duration_per_task_ms": 331039.8714859438, "avg_output_tokens_per_task": 22769, "last_run_timestamp": "2026-10-10T13:36:34.540Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6495237142857143, "accuracy_std_dev": 0.031240416581794835, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021353940000000002, "avg_duration_per_task_ms": 237904.40285714285, "avg_output_tokens_per_task": 10393.82, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.6480123486238532, "accuracy_std_dev": 0.024081519327223473, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.01123864036697248, "avg_duration_per_task_ms": 311115.2660550459, "avg_output_tokens_per_task": 17090.746788990826, "last_run_timestamp": "2026-10-10T18:41:07.171Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6480001000000001, "accuracy_std_dev": 0.02343796919722355, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.116462472, "avg_duration_per_task_ms": 144546.822, "avg_output_tokens_per_task": 12620.218, "last_run_timestamp": "2026-10-10T11:05:43.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6477778333333334, "accuracy_std_dev": 0.03430091421724631, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09916395333333333, "avg_duration_per_task_ms": 153090.92333333334, "avg_output_tokens_per_task": 12859.136666666667, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6476190000000001, "accuracy_std_dev": 0.02121044324316276, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02434872285714286, "avg_duration_per_task_ms": 154447.84, "avg_output_tokens_per_task": 10538.351428571428, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6473332, "accuracy_std_dev": 0.027840964224681578, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022036860000000002, "avg_duration_per_task_ms": 301895.372, "avg_output_tokens_per_task": 10556.636, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.6471734736842106, "accuracy_std_dev": 0.08517975279659948, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.10989668713450293, "avg_duration_per_task_ms": 1476007.2105263157, "avg_output_tokens_per_task": 80545.20760233919, "last_run_timestamp": "2026-10-10T19:03:17.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6470371111111111, "accuracy_std_dev": 0.03160113383846446, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03364751555555555, "avg_duration_per_task_ms": 80453.21555555555, "avg_output_tokens_per_task": 19157.16, "last_run_timestamp": "2026-10-10T18:38:13.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.646566095477387, "accuracy_std_dev": 0.12986162562313927, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.04628975628140704, "avg_duration_per_task_ms": 2127959.7989949747, "avg_output_tokens_per_task": 71164.3944723618, "last_run_timestamp": "2026-10-03T08:42:41.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6458334054054053, "accuracy_std_dev": 0.03153139496156523, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.013519689189189188, "avg_duration_per_task_ms": 1091817.445945946, "avg_output_tokens_per_task": 22703.493243243243, "last_run_timestamp": "2026-10-10T14:47:01.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.644208, "accuracy_std_dev": 0.03904293739234052, "run_count": 3, "total_questions": 141, "avg_cost_per_task_usd": 0.024240567375886524, "avg_duration_per_task_ms": 70321.02836879433, "avg_output_tokens_per_task": 14782.985815602837, "last_run_timestamp": "2026-10-10T18:42:24.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6433335, "accuracy_std_dev": 0.014272415770172448, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031060726666666667, "avg_duration_per_task_ms": 229987.72666666665, "avg_output_tokens_per_task": 16358.803333333333, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6433333333333333, "accuracy_std_dev": 0.14455548467015086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10531282, "avg_duration_per_task_ms": 1453976.54, "avg_output_tokens_per_task": 18878.353333333333, "last_run_timestamp": "2026-08-08T08:21:31.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.642570281124498, "accuracy_std_dev": 0.017333401911915625, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.014325088353413656, "avg_duration_per_task_ms": 874847.4096385542, "avg_output_tokens_per_task": 23432.22891566265, "last_run_timestamp": "2026-10-10T18:57:22.987Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.64166675, "accuracy_std_dev": 0.028625974026004797, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.017551492499999998, "avg_duration_per_task_ms": 331507.07, "avg_output_tokens_per_task": 20713.525, "last_run_timestamp": "2026-10-10T19:31:53.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.64, "accuracy_std_dev": 0.17706617541553699, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1395266857142857, "avg_duration_per_task_ms": 380475.96857142856, "avg_output_tokens_per_task": 8284.165714285715, "last_run_timestamp": "2026-10-10T12:10:04.065Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.639765898245614, "accuracy_std_dev": 0.0817052994894864, "run_count": 6, "total_questions": 285, "avg_cost_per_task_usd": 0.13215583157894736, "avg_duration_per_task_ms": 352439.649122807, "avg_output_tokens_per_task": 10044.143859649123, "last_run_timestamp": "2026-10-10T18:48:59.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.6383332500000001, "accuracy_std_dev": 0.028431218171008805, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.216643625, "avg_duration_per_task_ms": 1122861.59, "avg_output_tokens_per_task": 30036.18, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6378857133182845, "accuracy_std_dev": 0.03816624687028554, "run_count": 18, "total_questions": 886, "avg_cost_per_task_usd": 0.021538330699774268, "avg_duration_per_task_ms": 306224.9401805869, "avg_output_tokens_per_task": 18801.29683972912, "last_run_timestamp": "2026-10-10T19:10:56.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.04228062043326973, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08944130666666666, "avg_duration_per_task_ms": 118157.82666666666, "avg_output_tokens_per_task": 12612.513333333334, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.6346803232323233, "accuracy_std_dev": 0.01547599999999999, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.019095151515151516, "avg_duration_per_task_ms": 172977.9898989899, "avg_output_tokens_per_task": 9245.858585858587, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.6336735020408163, "accuracy_std_dev": 0.10052381662193295, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.02281844081632653, "avg_duration_per_task_ms": 906837.293877551, "avg_output_tokens_per_task": 30461.944897959183, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.631638420338983, "accuracy_std_dev": 0.06716132817452815, "run_count": 12, "total_questions": 590, "avg_cost_per_task_usd": 0.023603845762711864, "avg_duration_per_task_ms": 254670.11864406778, "avg_output_tokens_per_task": 24695.162711864406, "last_run_timestamp": "2026-10-10T19:04:00.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6314284285714286, "accuracy_std_dev": 0.018418242461965675, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.028592208571428572, "avg_duration_per_task_ms": 225611.1542857143, "avg_output_tokens_per_task": 10242.23142857143, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.631111, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02504141333333333, "avg_duration_per_task_ms": 168170.98, "avg_output_tokens_per_task": 11245.873333333333, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.63, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07672618, "avg_duration_per_task_ms": 231628.06, "avg_output_tokens_per_task": 10580.58, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6266667499999999, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.116286155, "avg_duration_per_task_ms": 669417.115, "avg_output_tokens_per_task": 30726.725, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6266665714285714, "accuracy_std_dev": 0.04047336572843623, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09534962571428571, "avg_duration_per_task_ms": 55134.33714285714, "avg_output_tokens_per_task": 3606.0057142857145, "last_run_timestamp": "2026-10-10T18:59:05.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.6266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015634330000000002, "avg_duration_per_task_ms": 100213.58, "avg_output_tokens_per_task": 19899.75, "last_run_timestamp": "2026-10-10T18:54:14.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.6266663333333334, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02545802, "avg_duration_per_task_ms": 175220.38666666666, "avg_output_tokens_per_task": 11691.16, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6216667499999999, "accuracy_std_dev": 0.04997217837144482, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08547625, "avg_duration_per_task_ms": 106037.93, "avg_output_tokens_per_task": 11996.815, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6174497919463087, "accuracy_std_dev": 0.011290615681716505, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.11355312416107383, "avg_duration_per_task_ms": 671627.1577181208, "avg_output_tokens_per_task": 32151.05033557047, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.61708325, "accuracy_std_dev": 0.02524513292077309, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.064975285, "avg_duration_per_task_ms": 31392.6825, "avg_output_tokens_per_task": 20231.8325, "last_run_timestamp": "2026-10-10T18:04:52.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.6155553333333333, "accuracy_std_dev": 0.04228062043326968, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09065885333333334, "avg_duration_per_task_ms": 148176.83333333334, "avg_output_tokens_per_task": 3889.28, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.613333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07264287999999999, "avg_duration_per_task_ms": 1482714.84, "avg_output_tokens_per_task": 23767.86, "last_run_timestamp": "2026-07-18T16:00:08.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6131220995475114, "accuracy_std_dev": 0.03551941611896371, "run_count": 9, "total_questions": 442, "avg_cost_per_task_usd": 0.041041904977375566, "avg_duration_per_task_ms": 84582.88235294117, "avg_output_tokens_per_task": 23671.278280542985, "last_run_timestamp": "2026-10-10T18:32:41.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.608, "accuracy_std_dev": 0.11488351975457575, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.058990216, "avg_duration_per_task_ms": 582295.388, "avg_output_tokens_per_task": 35784.112, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6076188571428572, "accuracy_std_dev": 0.03274303997331158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.046068719999999994, "avg_duration_per_task_ms": 191059.44, "avg_output_tokens_per_task": 3405.154285714286, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6060269955357143, "accuracy_std_dev": 0.032348691330028045, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.19772456473214287, "avg_duration_per_task_ms": 188670.52455357142, "avg_output_tokens_per_task": 13047.183035714286, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.605442, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.035960428571428574, "avg_duration_per_task_ms": 291757.14285714284, "avg_output_tokens_per_task": 21024.51020408163, "last_run_timestamp": "2026-10-10T19:16:28.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.6033332222222223, "accuracy_std_dev": 0.027397264890973577, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04748484000000001, "avg_duration_per_task_ms": 91610.96, "avg_output_tokens_per_task": 4621.971111111111, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.603333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.030717460000000002, "avg_duration_per_task_ms": 422658.18, "avg_output_tokens_per_task": 11680.98, "last_run_timestamp": "2026-07-18T16:00:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.600000125, "accuracy_std_dev": 0.06790519535064586, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036252275, "avg_duration_per_task_ms": 434224.3475, "avg_output_tokens_per_task": 19344.175, "last_run_timestamp": "2026-10-10T20:39:12.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5979899899497488, "accuracy_std_dev": 0.022095125194471644, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.020116060301507536, "avg_duration_per_task_ms": 100330.36180904522, "avg_output_tokens_per_task": 4353.20351758794, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.5933333333333334, "accuracy_std_dev": 0.02880349773127486, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21649743333333332, "avg_duration_per_task_ms": 191120.76666666666, "avg_output_tokens_per_task": 14122.313333333334, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.5913978951612903, "accuracy_std_dev": 0.012203964545998971, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.156355625, "avg_duration_per_task_ms": 554104.3709677419, "avg_output_tokens_per_task": 27726.576612903227, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5881481111111111, "accuracy_std_dev": 0.007554077566224961, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029889999999999996, "avg_duration_per_task_ms": 78610.16, "avg_output_tokens_per_task": 8108.122222222222, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.5856268495412844, "accuracy_std_dev": 0.2377490081349595, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.019255034862385324, "avg_duration_per_task_ms": 330742.1412844037, "avg_output_tokens_per_task": 9878.405504587156, "last_run_timestamp": "2026-10-10T12:29:44.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.5806451129032257, "accuracy_std_dev": 0.09293035051607196, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.038077282258064525, "avg_duration_per_task_ms": 2550353.040322581, "avg_output_tokens_per_task": 78807.90322580645, "last_run_timestamp": "2026-10-10T19:59:40.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.579037868556701, "accuracy_std_dev": 0.04024006578337068, "run_count": 8, "total_questions": 388, "avg_cost_per_task_usd": 0.07504983247422681, "avg_duration_per_task_ms": 90151.55927835051, "avg_output_tokens_per_task": 5664.682989690722, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.57416675, "accuracy_std_dev": 0.12703624537484373, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0368084625, "avg_duration_per_task_ms": 399880.175, "avg_output_tokens_per_task": 8103.195, "last_run_timestamp": "2026-10-10T19:10:33.654Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.572072, "accuracy_std_dev": 0.02796372233607127, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.09883952702702703, "avg_duration_per_task_ms": 105307.2027027027, "avg_output_tokens_per_task": 3660.560810810811, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5713488862876254, "accuracy_std_dev": 0.03528059746478538, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.011061280936454851, "avg_duration_per_task_ms": 252847.0033444816, "avg_output_tokens_per_task": 17983.45150501672, "last_run_timestamp": "2026-10-10T19:08:16.520Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5712907976878613, "accuracy_std_dev": 0.043538142708785955, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.18383380635838154, "avg_duration_per_task_ms": 1966033.9104046242, "avg_output_tokens_per_task": 40033.85260115607, "last_run_timestamp": "2026-10-10T19:23:34.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.571111, "accuracy_std_dev": 0.008314764498569157, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028662713333333336, "avg_duration_per_task_ms": 78035.85333333333, "avg_output_tokens_per_task": 8001.593333333333, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.5622223333333333, "accuracy_std_dev": 0.03456997912768952, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.23936231333333335, "avg_duration_per_task_ms": 165980.19333333333, "avg_output_tokens_per_task": 13303.673333333334, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5511113333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.035753513333333334, "avg_duration_per_task_ms": 61316.40666666667, "avg_output_tokens_per_task": 6659.566666666667, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5499152233502539, "accuracy_std_dev": 0.02918541097342131, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.0026568781725888327, "avg_duration_per_task_ms": 406357.32994923857, "avg_output_tokens_per_task": 6485.319796954314, "last_run_timestamp": "2026-10-10T19:11:30.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.54833325, "accuracy_std_dev": 0.09993065790305547, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0400913925, "avg_duration_per_task_ms": 97587.1625, "avg_output_tokens_per_task": 4289.61, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.5460636532663317, "accuracy_std_dev": 0.07718163585497526, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.05313004020100502, "avg_duration_per_task_ms": 465636.0527638191, "avg_output_tokens_per_task": 9251.170854271357, "last_run_timestamp": "2026-10-10T18:37:07.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5416667142857143, "accuracy_std_dev": 0.10219251042847255, "run_count": 7, "total_questions": 336, "avg_cost_per_task_usd": 0.06745207738095238, "avg_duration_per_task_ms": 446328.3333333333, "avg_output_tokens_per_task": 28269.869047619046, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5377905988372093, "accuracy_std_dev": 0.016893313972681363, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.02223077034883721, "avg_duration_per_task_ms": 1431254.4069767443, "avg_output_tokens_per_task": 108185.83430232559, "last_run_timestamp": "2026-10-10T19:31:51.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5333335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10219375, "avg_duration_per_task_ms": 375443.12, "avg_output_tokens_per_task": 17495.06, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.526667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02328844, "avg_duration_per_task_ms": 1276910.42, "avg_output_tokens_per_task": 153904.82, "last_run_timestamp": "2026-07-18T16:00:10.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.526667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03416076, "avg_duration_per_task_ms": 1845834.04, "avg_output_tokens_per_task": 105339.92, "last_run_timestamp": "2026-07-18T16:00:08.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5223215892857143, "accuracy_std_dev": 0.017182300915926027, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.06601706026785713, "avg_duration_per_task_ms": 301611.93526785716, "avg_output_tokens_per_task": 4026.2254464285716, "last_run_timestamp": "2026-10-10T18:28:10.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5221089591836734, "accuracy_std_dev": 0.023889008586586408, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.13644156632653062, "avg_duration_per_task_ms": 73125.92346938775, "avg_output_tokens_per_task": 2553.7397959183672, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.5188173145161291, "accuracy_std_dev": 0.2566565496323832, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.007278975806451614, "avg_duration_per_task_ms": 482880.30241935485, "avg_output_tokens_per_task": 15769.91935483871, "last_run_timestamp": "2026-10-10T18:54:19.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517500125, "accuracy_std_dev": 0.024930426803594342, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0282700725, "avg_duration_per_task_ms": 444815, "avg_output_tokens_per_task": 58343.125, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5156599127516779, "accuracy_std_dev": 0.03449671802746843, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.024214489932885906, "avg_duration_per_task_ms": 190286.47651006712, "avg_output_tokens_per_task": 48937.10738255033, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.511079063583815, "accuracy_std_dev": 0.04011160215484506, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.155033725433526, "avg_duration_per_task_ms": 1878039.6213872833, "avg_output_tokens_per_task": 35119.08670520231, "last_run_timestamp": "2026-10-10T19:22:46.804Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.5104164999999999, "accuracy_std_dev": 0.003472500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.024820281250000003, "avg_duration_per_task_ms": 480070.4583333333, "avg_output_tokens_per_task": 9578.8125, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5083332500000001, "accuracy_std_dev": 0.026404802805313633, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12635733, "avg_duration_per_task_ms": 53878.245, "avg_output_tokens_per_task": 2800.47, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.5055554444444444, "accuracy_std_dev": 0.03471227413117765, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02842421111111111, "avg_duration_per_task_ms": 279218.0311111111, "avg_output_tokens_per_task": 19993.748888888887, "last_run_timestamp": "2026-10-10T18:54:26.767Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.5045660136986302, "accuracy_std_dev": 0.02732875245760204, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.05861860273972602, "avg_duration_per_task_ms": 2642554.2671232875, "avg_output_tokens_per_task": 40736.36301369863, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5033670303030303, "accuracy_std_dev": 0.030340499999999992, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.01769851515151515, "avg_duration_per_task_ms": 135563.53535353535, "avg_output_tokens_per_task": 30988.575757575756, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.5033669747474748, "accuracy_std_dev": 0.027292446564651823, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.025276146464646466, "avg_duration_per_task_ms": 577739.3712121212, "avg_output_tokens_per_task": 53598.469696969696, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5013551951219513, "accuracy_std_dev": 0.01245431828082131, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.020900845528455284, "avg_duration_per_task_ms": 1272391.512195122, "avg_output_tokens_per_task": 112484.65853658537, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23109727999999996, "avg_duration_per_task_ms": 80168.37, "avg_output_tokens_per_task": 2746.03, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.48888899999999996, "accuracy_std_dev": 0.01872462563043652, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06696013, "avg_duration_per_task_ms": 315550.43333333335, "avg_output_tokens_per_task": 4089.9266666666667, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48833333333333334, "accuracy_std_dev": 0.018831396165859726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03323320666666667, "avg_duration_per_task_ms": 665027.6466666666, "avg_output_tokens_per_task": 61694.416666666664, "last_run_timestamp": "2026-10-03T16:04:32.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.4852150080645161, "accuracy_std_dev": 0.023420064714684302, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.028619552419354842, "avg_duration_per_task_ms": 386610.2439516129, "avg_output_tokens_per_task": 66355.3125, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.4842343648648649, "accuracy_std_dev": 0.02401659838297026, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.023849905405405405, "avg_duration_per_task_ms": 836568.3851351351, "avg_output_tokens_per_task": 22266.58108108108, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.48, "accuracy_std_dev": 0.0241687453956551, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034828557142857146, "avg_duration_per_task_ms": 43808.188571428575, "avg_output_tokens_per_task": 3663.8714285714286, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.4777776666666667, "accuracy_std_dev": 0.013698571612479242, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12510532, "avg_duration_per_task_ms": 53067.566666666666, "avg_output_tokens_per_task": 2709.2933333333335, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.4776285234899329, "accuracy_std_dev": 0.010657414972163225, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.1647015033557047, "avg_duration_per_task_ms": 2471417.852348993, "avg_output_tokens_per_task": 44160.27181208054, "last_run_timestamp": "2026-10-03T16:12:22.579Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.47583349999999996, "accuracy_std_dev": 0.03650536870790377, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02416941, "avg_duration_per_task_ms": 59754.7125, "avg_output_tokens_per_task": 3775.0125, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.47333325000000004, "accuracy_std_dev": 0.025603927651583064, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0251383825, "avg_duration_per_task_ms": 126301.635, "avg_output_tokens_per_task": 3791.835, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03429832, "avg_duration_per_task_ms": 458175.58, "avg_output_tokens_per_task": 37835.14, "last_run_timestamp": "2026-07-18T16:00:08.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.4728994552845529, "accuracy_std_dev": 0.04759788478073367, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.02759002032520325, "avg_duration_per_task_ms": 2332100.9227642277, "avg_output_tokens_per_task": 138904.24390243902, "last_run_timestamp": "2026-10-10T19:06:47.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4724461814516129, "accuracy_std_dev": 0.0356696342375416, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.015281346774193549, "avg_duration_per_task_ms": 122771.32258064517, "avg_output_tokens_per_task": 3293.3064516129034, "last_run_timestamp": "2026-10-10T18:31:25.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4723047626112759, "accuracy_std_dev": 0.03812044880134972, "run_count": 7, "total_questions": 337, "avg_cost_per_task_usd": 0.05108310979228487, "avg_duration_per_task_ms": 212808.3293768546, "avg_output_tokens_per_task": 93380.04451038576, "last_run_timestamp": "2026-10-10T18:37:46.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.472222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.02306027083333333, "avg_duration_per_task_ms": 25624.729166666668, "avg_output_tokens_per_task": 1174.2916666666667, "last_run_timestamp": "2026-07-18T16:00:08.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.47047614285714284, "accuracy_std_dev": 0.020426164806293848, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021544591428571434, "avg_duration_per_task_ms": 680725.9685714286, "avg_output_tokens_per_task": 21489.04857142857, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.46952399999999994, "accuracy_std_dev": 0.034523096873170074, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.018877348571428573, "avg_duration_per_task_ms": 90973.66571428571, "avg_output_tokens_per_task": 3606.62, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.46833325000000003, "accuracy_std_dev": 0.03295620410389368, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09710940000000001, "avg_duration_per_task_ms": 99117.8225, "avg_output_tokens_per_task": 2749.945, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.4662164729729729, "accuracy_std_dev": 0.03892660093903227, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04219703378378378, "avg_duration_per_task_ms": 711257.6756756756, "avg_output_tokens_per_task": 98782.22297297297, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.465926, "accuracy_std_dev": 0.03230940398363575, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.022285648888888884, "avg_duration_per_task_ms": 222200.71555555557, "avg_output_tokens_per_task": 3598.428888888889, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4608501677852349, "accuracy_std_dev": 0.034994868451198195, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04671855033557047, "avg_duration_per_task_ms": 767600.1241610738, "avg_output_tokens_per_task": 101728.0033557047, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.46068351282051284, "accuracy_std_dev": 0.04225414673955088, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.01900871794871795, "avg_duration_per_task_ms": 170098.1794871795, "avg_output_tokens_per_task": 35854.89743589744, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.45814811111111114, "accuracy_std_dev": 0.03865370011164504, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05284759333333334, "avg_duration_per_task_ms": 162308.54, "avg_output_tokens_per_task": 3744.4333333333334, "last_run_timestamp": "2026-10-10T19:04:15.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4577777777777778, "accuracy_std_dev": 0.04455537267959145, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05422914222222222, "avg_duration_per_task_ms": 126062.82666666666, "avg_output_tokens_per_task": 3580.0733333333333, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.4570707575757576, "accuracy_std_dev": 0.044267226719521297, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.05378886363636363, "avg_duration_per_task_ms": 73384.54040404041, "avg_output_tokens_per_task": 3408.1742424242425, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.4560182083333333, "accuracy_std_dev": 0.022912149746368182, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.08057918055555555, "avg_duration_per_task_ms": 52530.756944444445, "avg_output_tokens_per_task": 3473.4930555555557, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4511785656565656, "accuracy_std_dev": 0.03515039216708685, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04521708585858586, "avg_duration_per_task_ms": 779859.6717171717, "avg_output_tokens_per_task": 105433.07575757576, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.45095228571428575, "accuracy_std_dev": 0.02741314960126298, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.020993765714285715, "avg_duration_per_task_ms": 566234.7285714286, "avg_output_tokens_per_task": 21154.594285714287, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.45, "accuracy_std_dev": 0.01784692109767581, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03456686, "avg_duration_per_task_ms": 476526.7733333333, "avg_output_tokens_per_task": 23360.6, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.44750012499999997, "accuracy_std_dev": 0.06501608615073176, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.16146158749999998, "avg_duration_per_task_ms": 120680.0125, "avg_output_tokens_per_task": 3281.1225, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.447037, "accuracy_std_dev": 0.02899323273915255, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04068735111111112, "avg_duration_per_task_ms": 283005.94444444444, "avg_output_tokens_per_task": 3628.5377777777776, "last_run_timestamp": "2026-10-10T19:42:35.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.4444443333333334, "accuracy_std_dev": 0.030952277087298266, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03472188, "avg_duration_per_task_ms": 283560.81333333335, "avg_output_tokens_per_task": 21592.06, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4441665, "accuracy_std_dev": 0.023555566693459103, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03439918, "avg_duration_per_task_ms": 113438.9075, "avg_output_tokens_per_task": 3146.3175, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.44333325, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025700495, "avg_duration_per_task_ms": 61023.305, "avg_output_tokens_per_task": 2441.375, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.44186027325581395, "accuracy_std_dev": 0.0336023614676719, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.05302273255813953, "avg_duration_per_task_ms": 228343.90697674418, "avg_output_tokens_per_task": 2116.357558139535, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4416665, "accuracy_std_dev": 0.017873086533948188, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.104832345, "avg_duration_per_task_ms": 890298.59, "avg_output_tokens_per_task": 21334.155, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.4408060680100756, "accuracy_std_dev": 0.12174978780973233, "run_count": 16, "total_questions": 794, "avg_cost_per_task_usd": 0.03954040302267003, "avg_duration_per_task_ms": 655917.3727959698, "avg_output_tokens_per_task": 177113.59697733, "last_run_timestamp": "2026-10-10T18:58:33.701Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2288897, "avg_duration_per_task_ms": 83836.68, "avg_output_tokens_per_task": 2734.56, "last_run_timestamp": "2026-07-18T08:04:08.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.43869733333333333, "accuracy_std_dev": 0.023050038409813497, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.01657819540229885, "avg_duration_per_task_ms": 200645.3132183908, "avg_output_tokens_per_task": 2785.6264367816093, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4383965367088608, "accuracy_std_dev": 0.03960836597472182, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.05240046329113924, "avg_duration_per_task_ms": 422067.2658227848, "avg_output_tokens_per_task": 20533.427848101266, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.43777799999999994, "accuracy_std_dev": 0.03861798340151904, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.19450383999999998, "avg_duration_per_task_ms": 50798.87333333334, "avg_output_tokens_per_task": 2047.5333333333333, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.011330936570096718, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02595394666666667, "avg_duration_per_task_ms": 66273.3, "avg_output_tokens_per_task": 2407.2933333333335, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21387673999999998, "avg_duration_per_task_ms": 47267.2, "avg_output_tokens_per_task": 2224.56, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.4372223333333333, "accuracy_std_dev": 0.025270150254576823, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.027693089999999997, "avg_duration_per_task_ms": 78315.11, "avg_output_tokens_per_task": 2793.7166666666667, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.4362416308724832, "accuracy_std_dev": 0.02031037140436711, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.0464321610738255, "avg_duration_per_task_ms": 136873.4798657718, "avg_output_tokens_per_task": 2090.986577181208, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.4358108783783784, "accuracy_std_dev": 0.3084304226783661, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.015068885135135136, "avg_duration_per_task_ms": 91458.4831081081, "avg_output_tokens_per_task": 15700.83445945946, "last_run_timestamp": "2026-10-10T18:32:50.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.435, "accuracy_std_dev": 0.020749805890658335, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.074401825, "avg_duration_per_task_ms": 778012.545, "avg_output_tokens_per_task": 20313.29, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.43434354545454545, "accuracy_std_dev": 0.022037645087440712, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.06906736868686869, "avg_duration_per_task_ms": 323353.22222222225, "avg_output_tokens_per_task": 3378.3585858585857, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.43416675, "accuracy_std_dev": 0.04222137919570487, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.400907955, "avg_duration_per_task_ms": 107994.6975, "avg_output_tokens_per_task": 3151.7075, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43416662500000003, "accuracy_std_dev": 0.01613729505630899, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2170375225, "avg_duration_per_task_ms": 46329.7025, "avg_output_tokens_per_task": 2265.4175, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4337431393442623, "accuracy_std_dev": 0.021677717946315293, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.032824139344262294, "avg_duration_per_task_ms": 478154.40573770495, "avg_output_tokens_per_task": 4694.745901639344, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.431111, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029769013333333334, "avg_duration_per_task_ms": 630152.1266666667, "avg_output_tokens_per_task": 18760.18666666667, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.43089426016260163, "accuracy_std_dev": 0.03483555910043644, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.04202976422764227, "avg_duration_per_task_ms": 58240.52032520325, "avg_output_tokens_per_task": 2134.682926829268, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.43083325, "accuracy_std_dev": 0.031213896924086562, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029063365, "avg_duration_per_task_ms": 761962.965, "avg_output_tokens_per_task": 19001.86, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.24811575000000002, "avg_duration_per_task_ms": 46287.18, "avg_output_tokens_per_task": 2565.8, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.42666699999999996, "accuracy_std_dev": 0.04000000000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.031438360000000005, "avg_duration_per_task_ms": 468393.89, "avg_output_tokens_per_task": 22584.78, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.425833375, "accuracy_std_dev": 0.013616710018737092, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.024625232500000004, "avg_duration_per_task_ms": 228841.615, "avg_output_tokens_per_task": 3561.265, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.4247621428571428, "accuracy_std_dev": 0.04058525139022997, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05475465714285714, "avg_duration_per_task_ms": 141536.79714285713, "avg_output_tokens_per_task": 3538.397142857143, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.42372316935483867, "accuracy_std_dev": 0.031047363202694044, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.025069254032258063, "avg_duration_per_task_ms": 357367.3891129032, "avg_output_tokens_per_task": 3605.673387096774, "last_run_timestamp": "2026-10-10T19:22:29.626Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.42144822950819677, "accuracy_std_dev": 0.025118027628776907, "run_count": 10, "total_questions": 488, "avg_cost_per_task_usd": 0.050889221311475406, "avg_duration_per_task_ms": 182960.04508196723, "avg_output_tokens_per_task": 4075.3032786885246, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.4170453954545455, "accuracy_std_dev": 0.02305814667528085, "run_count": 9, "total_questions": 440, "avg_cost_per_task_usd": 0.031998124999999995, "avg_duration_per_task_ms": 704757.9227272727, "avg_output_tokens_per_task": 7358.236363636363, "last_run_timestamp": "2026-10-10T18:59:12.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4166667045454545, "accuracy_std_dev": 0.029461897667544087, "run_count": 9, "total_questions": 440, "avg_cost_per_task_usd": 0.011195077272727273, "avg_duration_per_task_ms": 112204.60909090909, "avg_output_tokens_per_task": 3271.315909090909, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40777783333333334, "accuracy_std_dev": 0.01781244190087991, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025491406666666664, "avg_duration_per_task_ms": 73232.91333333333, "avg_output_tokens_per_task": 2860.7966666666666, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.40541675, "accuracy_std_dev": 0.021209148643627825, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06368001, "avg_duration_per_task_ms": 101217.8175, "avg_output_tokens_per_task": 2945.865, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.4053599698492462, "accuracy_std_dev": 0.008101744842933528, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.06672347236180905, "avg_duration_per_task_ms": 212067.50753768845, "avg_output_tokens_per_task": 8080.718592964824, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.4051852222222222, "accuracy_std_dev": 0.12207173923538002, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02358350666666667, "avg_duration_per_task_ms": 188827.18, "avg_output_tokens_per_task": 3348.0044444444443, "last_run_timestamp": "2026-10-10T18:35:31.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.40333325000000003, "accuracy_std_dev": 0.034960136029304874, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05923553749999999, "avg_duration_per_task_ms": 320941.4325, "avg_output_tokens_per_task": 7834.6225, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.402500125, "accuracy_std_dev": 0.02559013346212511, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.026921807500000002, "avg_duration_per_task_ms": 293558.7475, "avg_output_tokens_per_task": 4184.715, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.400833375, "accuracy_std_dev": 0.029332849192234545, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0642667, "avg_duration_per_task_ms": 297648.6875, "avg_output_tokens_per_task": 11410.885, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.40083312500000007, "accuracy_std_dev": 0.02459377320805766, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07675446250000001, "avg_duration_per_task_ms": 122147.305, "avg_output_tokens_per_task": 3115.835, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.398333375, "accuracy_std_dev": 0.05074458509766707, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.048334209999999996, "avg_duration_per_task_ms": 87716.1125, "avg_output_tokens_per_task": 3723.8325, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.39726041780821914, "accuracy_std_dev": 0.26615553243962864, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.011427219178082192, "avg_duration_per_task_ms": 86701.55479452055, "avg_output_tokens_per_task": 2538.931506849315, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.396667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0528661, "avg_duration_per_task_ms": 135118.22, "avg_output_tokens_per_task": 7825.32, "last_run_timestamp": "2026-07-25T08:07:19.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.39521795358649786, "accuracy_std_dev": 0.015940999687598014, "run_count": 5, "total_questions": 237, "avg_cost_per_task_usd": 0.009386248945147679, "avg_duration_per_task_ms": 65821.63713080168, "avg_output_tokens_per_task": 1124.5696202531647, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.3944446666666666, "accuracy_std_dev": 0.015947545189011245, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07351260000000001, "avg_duration_per_task_ms": 297595.3333333333, "avg_output_tokens_per_task": 7950.953333333333, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.39249987500000005, "accuracy_std_dev": 0.016811927398706402, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0369896575, "avg_duration_per_task_ms": 191395.045, "avg_output_tokens_per_task": 3738.2225, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.39210027167630057, "accuracy_std_dev": 0.08499299536756556, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.3060756098265896, "avg_duration_per_task_ms": 324526.6416184971, "avg_output_tokens_per_task": 12470.086705202311, "last_run_timestamp": "2026-10-10T18:31:45.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3912555560538117, "accuracy_std_dev": 0.029116472984579173, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.059476125560538114, "avg_duration_per_task_ms": 375172.7399103139, "avg_output_tokens_per_task": 3892.973094170404, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.39, "accuracy_std_dev": 0.014782396941407474, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028387520000000006, "avg_duration_per_task_ms": 78569.82333333333, "avg_output_tokens_per_task": 4258.516666666666, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.389166625, "accuracy_std_dev": 0.02355558880572453, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.025453972499999998, "avg_duration_per_task_ms": 227372.8075, "avg_output_tokens_per_task": 4499.2425, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.38666671428571425, "accuracy_std_dev": 0.008728715610219052, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014610905714285714, "avg_duration_per_task_ms": 90807.17428571428, "avg_output_tokens_per_task": 601.44, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.37776065177195683, "accuracy_std_dev": 0.01772313217238985, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.012601627118644071, "avg_duration_per_task_ms": 36693.28659476117, "avg_output_tokens_per_task": 655.8120184899846, "last_run_timestamp": "2026-10-10T18:35:37.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.37583325, "accuracy_std_dev": 0.03264919388265964, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.051693865, "avg_duration_per_task_ms": 72470.795, "avg_output_tokens_per_task": 4363.9325, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.373333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07157664, "avg_duration_per_task_ms": 16604.76, "avg_output_tokens_per_task": 863.76, "last_run_timestamp": "2026-07-18T16:00:09.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.373333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.030392839999999997, "avg_duration_per_task_ms": 86392.54, "avg_output_tokens_per_task": 5907.74, "last_run_timestamp": "2026-07-18T08:03:14.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.37185199999999996, "accuracy_std_dev": 0.07211295476003061, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.006007911111111111, "avg_duration_per_task_ms": 147635.88666666666, "avg_output_tokens_per_task": 3781.9444444444443, "last_run_timestamp": "2026-10-10T19:09:43.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.3718273705583756, "accuracy_std_dev": 0.06753307245238718, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.0041109720812182735, "avg_duration_per_task_ms": 291217.7411167513, "avg_output_tokens_per_task": 4238.002538071066, "last_run_timestamp": "2026-10-10T19:11:16.157Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.3714284285714286, "accuracy_std_dev": 0.017715366073046326, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03264426571428572, "avg_duration_per_task_ms": 156269.61714285714, "avg_output_tokens_per_task": 4433.125714285714, "last_run_timestamp": "2026-10-03T16:07:02.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.35762146733668343, "accuracy_std_dev": 0.20389728295072376, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.12249766080402011, "avg_duration_per_task_ms": 301830.15075376886, "avg_output_tokens_per_task": 40654.94472361809, "last_run_timestamp": "2026-10-10T18:31:25.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.3575, "accuracy_std_dev": 0.033819176021156985, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07891495999999999, "avg_duration_per_task_ms": 25792.25, "avg_output_tokens_per_task": 920.17, "last_run_timestamp": "2026-10-10T18:49:13.368Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.3567604417670683, "accuracy_std_dev": 0.024567950668299542, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.0795283875502008, "avg_duration_per_task_ms": 211758.7168674699, "avg_output_tokens_per_task": 7341.082329317269, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.3566663333333333, "accuracy_std_dev": 0.016996731711975962, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.043029019999999994, "avg_duration_per_task_ms": 111012.42666666667, "avg_output_tokens_per_task": 4511.693333333334, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35592184615384614, "accuracy_std_dev": 0.02760835600602419, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.07156286996336997, "avg_duration_per_task_ms": 377710.5695970696, "avg_output_tokens_per_task": 7880.789377289378, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.3367119324324324, "accuracy_std_dev": 0.23078482961913535, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.014007192567567568, "avg_duration_per_task_ms": 1521898.9797297297, "avg_output_tokens_per_task": 94037.88851351352, "last_run_timestamp": "2026-10-03T16:12:24.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.33666633333333335, "accuracy_std_dev": 0.004714045207910321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029078933333333334, "avg_duration_per_task_ms": 79568.04666666666, "avg_output_tokens_per_task": 3568.733333333333, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.3340808340807174, "accuracy_std_dev": 0.05005079503642052, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.029606439461883405, "avg_duration_per_task_ms": 98144.11883408071, "avg_output_tokens_per_task": 1206.6883408071749, "last_run_timestamp": "2026-10-10T19:14:33.599Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.3296970909090909, "accuracy_std_dev": 0.023845300316279384, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08440126545454546, "avg_duration_per_task_ms": 151545.54545454544, "avg_output_tokens_per_task": 8175.638181818182, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.30896692982456136, "accuracy_std_dev": 0.03129504337294144, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.01647059649122807, "avg_duration_per_task_ms": 365014.59064327483, "avg_output_tokens_per_task": 4454.833333333333, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.3022222222222222, "accuracy_std_dev": 0.036243388499843904, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16888293333333335, "avg_duration_per_task_ms": 128280.22444444445, "avg_output_tokens_per_task": 1888.62, "last_run_timestamp": "2026-10-10T19:11:13.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.2921211818181818, "accuracy_std_dev": 0.32697940222083444, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.017488016363636363, "avg_duration_per_task_ms": 53819.056363636366, "avg_output_tokens_per_task": 9497.68909090909, "last_run_timestamp": "2026-10-10T20:16:03.339Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.2844443333333333, "accuracy_std_dev": 0.031894788459698065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02081186, "avg_duration_per_task_ms": 62184.12666666666, "avg_output_tokens_per_task": 3804.7533333333336, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.28000025, "accuracy_std_dev": 0.027080189561513406, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02535222, "avg_duration_per_task_ms": 67904.845, "avg_output_tokens_per_task": 4023.05, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.2797619910714286, "accuracy_std_dev": 0.1899431271985133, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.15921619642857143, "avg_duration_per_task_ms": 942346.96875, "avg_output_tokens_per_task": 6098.834821428572, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2622223333333333, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08337879999999999, "avg_duration_per_task_ms": 576772.3666666667, "avg_output_tokens_per_task": 5220.566666666667, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.260417, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.027361125, "avg_duration_per_task_ms": 394399.5416666667, "avg_output_tokens_per_task": 4477.166666666667, "last_run_timestamp": "2026-07-18T08:04:09.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02790544, "avg_duration_per_task_ms": 464213.12, "avg_output_tokens_per_task": 5425.66, "last_run_timestamp": "2026-07-18T08:04:25.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24200919178082192, "accuracy_std_dev": 0.027381797668443095, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.07592458219178082, "avg_duration_per_task_ms": 576526.0342465753, "avg_output_tokens_per_task": 5262.068493150685, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.2325, "accuracy_std_dev": 0.06851085072271398, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.013581069999999997, "avg_duration_per_task_ms": 21748.2225, "avg_output_tokens_per_task": 1125.92, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.19666649999999997, "accuracy_std_dev": 0.023804971471312466, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00425299, "avg_duration_per_task_ms": 15962.445, "avg_output_tokens_per_task": 1579.045, "last_run_timestamp": "2026-10-03T16:07:44.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.18791944295302013, "accuracy_std_dev": 0.10708233389741122, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.20449519798657714, "avg_duration_per_task_ms": 608397.9966442953, "avg_output_tokens_per_task": 10862.70134228188, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.18766967073170732, "accuracy_std_dev": 0.018396157170452745, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.011055426829268291, "avg_duration_per_task_ms": 138140.34552845528, "avg_output_tokens_per_task": 5390.59756097561, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.17999985714285713, "accuracy_std_dev": 0.12173356543044388, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.013454491428571429, "avg_duration_per_task_ms": 61907.58571428571, "avg_output_tokens_per_task": 1212.5742857142857, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.13333333333333333, "accuracy_std_dev": 0.021430473285072877, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007961373333333334, "avg_duration_per_task_ms": 80728.30333333333, "avg_output_tokens_per_task": 2469.17, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.11833325, "accuracy_std_dev": 0.018484159885899604, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007158584999999999, "avg_duration_per_task_ms": 55526.67, "avg_output_tokens_per_task": 2353.445, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.108889, "accuracy_std_dev": 0.008314764498569193, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006775626666666668, "avg_duration_per_task_ms": 60620.74, "avg_output_tokens_per_task": 2327.0066666666667, "last_run_timestamp": "2026-10-01T19:04:36.194Z" } ] }