{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-10-04", "retrieved_at": "2026-10-04T01:45:32.205Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2110, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9483723333333332, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20306945454545455, "avg_duration_per_task_ms": 134835.03535353535, "avg_output_tokens_per_task": 16798.409090909092, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021931015151515153, "avg_duration_per_task_ms": 68600.9191919192, "avg_output_tokens_per_task": 1835.4242424242425, "last_run_timestamp": "2026-10-03T01:13:05.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.947009, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.07090916923076923, "avg_duration_per_task_ms": 142045.61538461538, "avg_output_tokens_per_task": 18762.907692307694, "last_run_timestamp": "2026-10-01T08:30:23.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11987038888888889, "avg_duration_per_task_ms": 35572.169191919194, "avg_output_tokens_per_task": 2020.9419191919192, "last_run_timestamp": "2026-10-01T08:24:02.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.015926459595959598, "avg_duration_per_task_ms": 24665.858585858587, "avg_output_tokens_per_task": 2219.030303030303, "last_run_timestamp": "2026-09-28T18:11:34.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.9393935, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.29793838636363634, "avg_duration_per_task_ms": 106861.7196969697, "avg_output_tokens_per_task": 11692.656565656565, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.937943329787234, "accuracy_std_dev": 0.011200313606720516, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.023761370567375886, "avg_duration_per_task_ms": 52418.78723404255, "avg_output_tokens_per_task": 7636.52304964539, "last_run_timestamp": "2026-10-01T16:16:37.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9371493333333333, "accuracy_std_dev": 0.004199386807089248, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.30925494444444446, "avg_duration_per_task_ms": 152328.37037037036, "avg_output_tokens_per_task": 10179.092592592593, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9339223333333333, "accuracy_std_dev": 0.0062945671724820625, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.13663711026936026, "avg_duration_per_task_ms": 176815.70622895623, "avg_output_tokens_per_task": 8900.839225589225, "last_run_timestamp": "2026-10-03T10:36:57.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.9337823333333333, "accuracy_std_dev": 0.008398773614178557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3842955404040404, "avg_duration_per_task_ms": 82918.1936026936, "avg_output_tokens_per_task": 11996.59090909091, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9332206666666667, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.24476150168350166, "avg_duration_per_task_ms": 101605.01851851853, "avg_output_tokens_per_task": 11508.424242424242, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9326596250000001, "accuracy_std_dev": 0.00862529761714779, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.32610833964646463, "avg_duration_per_task_ms": 161836.85416666666, "avg_output_tokens_per_task": 10112.390782828283, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.9326595000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02474653787878788, "avg_duration_per_task_ms": 20002.444444444445, "avg_output_tokens_per_task": 2234.5, "last_run_timestamp": "2026-10-03T09:05:12.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9325395714285714, "accuracy_std_dev": 0.004401689845956867, "run_count": 4, "total_questions": 756, "avg_cost_per_task_usd": 0.06429072222222224, "avg_duration_per_task_ms": 63071.10317460317, "avg_output_tokens_per_task": 11529.871693121693, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.931257, "accuracy_std_dev": 0.00924235327175929, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.026367194444444447, "avg_duration_per_task_ms": 19064.280303030304, "avg_output_tokens_per_task": 2249.5185185185187, "last_run_timestamp": "2026-10-03T09:39:59.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.930415, "accuracy_std_dev": 0.004827415699246697, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06216489141414142, "avg_duration_per_task_ms": 64761.851851851854, "avg_output_tokens_per_task": 11730.429292929293, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9278394565075213, "accuracy_std_dev": 0.008041238261262993, "run_count": 25, "total_questions": 4587, "avg_cost_per_task_usd": 0.1608165642031829, "avg_duration_per_task_ms": 149318.05646391978, "avg_output_tokens_per_task": 8864.42119032047, "last_run_timestamp": "2026-10-03T10:48:13.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9277417729032257, "accuracy_std_dev": 0.015966818205265555, "run_count": 5, "total_questions": 775, "avg_cost_per_task_usd": 0.09847772, "avg_duration_per_task_ms": 244962.84516129032, "avg_output_tokens_per_task": 7582.7083870967745, "last_run_timestamp": "2026-10-03T10:05:38.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.004418715424192887, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05958464141414141, "avg_duration_per_task_ms": 58983.43771043771, "avg_output_tokens_per_task": 11740.584175084176, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.926400137295082, "accuracy_std_dev": 0.007132075389394045, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09721093032786886, "avg_duration_per_task_ms": 197311.04713114753, "avg_output_tokens_per_task": 8911.119877049181, "last_run_timestamp": "2026-10-03T10:09:17.601Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.925926, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025347974747474744, "avg_duration_per_task_ms": 21521.87373737374, "avg_output_tokens_per_task": 2298.931818181818, "last_run_timestamp": "2026-10-03T09:35:22.953Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.925926, "accuracy_std_dev": 0.005498287975967322, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1441699393939394, "avg_duration_per_task_ms": 61658.986531986535, "avg_output_tokens_per_task": 15894.8367003367, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.925926, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.14454752525252526, "avg_duration_per_task_ms": 66158.0808080808, "avg_output_tokens_per_task": 9383.212121212122, "last_run_timestamp": "2026-10-03T19:50:41.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9259256666666668, "accuracy_std_dev": 0.0036366213562713575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1413088872053872, "avg_duration_per_task_ms": 82468.60606060606, "avg_output_tokens_per_task": 15584.752525252525, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9259255000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025620611111111115, "avg_duration_per_task_ms": 19140.497474747473, "avg_output_tokens_per_task": 2323.35101010101, "last_run_timestamp": "2026-10-03T09:34:06.834Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9255440309278352, "accuracy_std_dev": 0.0047577793839843644, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.3407643092783505, "avg_duration_per_task_ms": 132309.46391752578, "avg_output_tokens_per_task": 10197.671821305841, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9248735, "accuracy_std_dev": 0.00862277742377712, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10364464204545455, "avg_duration_per_task_ms": 55123.33648989899, "avg_output_tokens_per_task": 3517.8642676767677, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9242421666666667, "accuracy_std_dev": 0.0048599613653014136, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.14878865656565657, "avg_duration_per_task_ms": 85040.50420875421, "avg_output_tokens_per_task": 15641.649831649831, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto", "model_slug": "unbiased/pareto", "model_name": "Pareto", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03377419696969697, "avg_duration_per_task_ms": 114763.54545454546, "avg_output_tokens_per_task": 4358.363636363636, "last_run_timestamp": "2026-09-29T03:11:33.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.008625346413333228, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05533751388888889, "avg_duration_per_task_ms": 61705.84722222222, "avg_output_tokens_per_task": 3564.286616161616, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9227054347826086, "accuracy_std_dev": 0.0037342422507786015, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.09927257048748353, "avg_duration_per_task_ms": 52555.73781291173, "avg_output_tokens_per_task": 3495.6416337285905, "last_run_timestamp": "2026-10-01T19:12:57.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9219427970033833, "accuracy_std_dev": 0.006990580536078893, "run_count": 11, "total_questions": 2069, "avg_cost_per_task_usd": 0.15209735766070562, "avg_duration_per_task_ms": 132397.76655389077, "avg_output_tokens_per_task": 9993.401159980667, "last_run_timestamp": "2026-10-03T10:21:23.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9212965, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13693505808080808, "avg_duration_per_task_ms": 189958.2702020202, "avg_output_tokens_per_task": 8925.60101010101, "last_run_timestamp": "2026-10-03T09:18:30.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06043574579124578, "avg_duration_per_task_ms": 62971.38047138047, "avg_output_tokens_per_task": 3586.8703703703704, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.920875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025261131313131315, "avg_duration_per_task_ms": 21541.484848484848, "avg_output_tokens_per_task": 2294.9949494949497, "last_run_timestamp": "2026-10-03T09:35:43.283Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9205012222222222, "accuracy_std_dev": 0.00850227984821038, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1576766610549944, "avg_duration_per_task_ms": 236410.12177328844, "avg_output_tokens_per_task": 12595.254208754208, "last_run_timestamp": "2026-09-26T08:55:21.775Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.920314, "accuracy_std_dev": 0.007316769300722847, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1692038207070707, "avg_duration_per_task_ms": 320867.6077441077, "avg_output_tokens_per_task": 9628.20622895623, "last_run_timestamp": "2026-10-03T09:31:32.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026391550505050505, "avg_duration_per_task_ms": 26392.646464646463, "avg_output_tokens_per_task": 2240.712121212121, "last_run_timestamp": "2026-10-01T08:37:17.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9185433141891891, "accuracy_std_dev": 0.011935651348422994, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.13230360641891895, "avg_duration_per_task_ms": 318469.4380630631, "avg_output_tokens_per_task": 10386.162725225226, "last_run_timestamp": "2026-10-03T09:44:31.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9181816, "accuracy_std_dev": 0.007721961838807562, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0816559797979798, "avg_duration_per_task_ms": 327979.47777777776, "avg_output_tokens_per_task": 24710.48888888889, "last_run_timestamp": "2026-10-03T11:44:39.356Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9173829326647566, "accuracy_std_dev": 0.013022336741090185, "run_count": 11, "total_questions": 2094, "avg_cost_per_task_usd": 0.12426453438395414, "avg_duration_per_task_ms": 122014.65329512894, "avg_output_tokens_per_task": 8324.876313276027, "last_run_timestamp": "2026-10-03T09:17:37.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9171931536842105, "accuracy_std_dev": 0.011022305393496383, "run_count": 4, "total_questions": 475, "avg_cost_per_task_usd": 0.10596987368421053, "avg_duration_per_task_ms": 176112.00210526315, "avg_output_tokens_per_task": 6923.242105263158, "last_run_timestamp": "2026-10-03T10:04:15.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9161053333333333, "accuracy_std_dev": 0.004900048423797018, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10286850841750843, "avg_duration_per_task_ms": 630668.6658249159, "avg_output_tokens_per_task": 39966.40488215488, "last_run_timestamp": "2026-10-03T09:15:20.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9158830997830802, "accuracy_std_dev": 0.0087081660666812, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.2615176702819957, "avg_duration_per_task_ms": 379032.86912509036, "avg_output_tokens_per_task": 37727.92263195951, "last_run_timestamp": "2026-10-03T09:17:56.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9153725336438923, "accuracy_std_dev": 0.007489152329202574, "run_count": 10, "total_questions": 1932, "avg_cost_per_task_usd": 0.14073902795031057, "avg_duration_per_task_ms": 293484.7536231884, "avg_output_tokens_per_task": 9227.590062111802, "last_run_timestamp": "2026-10-03T10:17:13.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9148461928609483, "accuracy_std_dev": 0.040300990523310966, "run_count": 10, "total_questions": 1877, "avg_cost_per_task_usd": 0.11435862759722962, "avg_duration_per_task_ms": 215504.55833777305, "avg_output_tokens_per_task": 9109.937133724028, "last_run_timestamp": "2026-10-03T10:08:37.048Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9145619999999999, "accuracy_std_dev": 0.003005841230005351, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025954439393939394, "avg_duration_per_task_ms": 19982.071969696968, "avg_output_tokens_per_task": 2234.438131313131, "last_run_timestamp": "2026-10-03T10:12:04.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9141412499999999, "accuracy_std_dev": 0.012579376886561864, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03262016877104377, "avg_duration_per_task_ms": 228754.26725589225, "avg_output_tokens_per_task": 26998.277356902356, "last_run_timestamp": "2026-10-03T16:51:25.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9136839532631578, "accuracy_std_dev": 0.030012698133341075, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.017909636210526318, "avg_duration_per_task_ms": 57780.396631578944, "avg_output_tokens_per_task": 3003.1595789473686, "last_run_timestamp": "2026-09-30T03:11:21.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9132995, "accuracy_std_dev": 0.010546877796295947, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.15848487878787879, "avg_duration_per_task_ms": 520006.99494949495, "avg_output_tokens_per_task": 39704.125, "last_run_timestamp": "2026-10-03T09:15:53.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9117844, "accuracy_std_dev": 0.006079420781620585, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03790721616161616, "avg_duration_per_task_ms": 129456.97070707071, "avg_output_tokens_per_task": 28484.87575757576, "last_run_timestamp": "2026-10-03T09:28:48.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9115787584210527, "accuracy_std_dev": 0.03150249392869556, "run_count": 20, "total_questions": 3800, "avg_cost_per_task_usd": 0.14629828526315786, "avg_duration_per_task_ms": 443992.2734210526, "avg_output_tokens_per_task": 8936.269473684211, "last_run_timestamp": "2026-10-03T10:26:01.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9098249433962265, "accuracy_std_dev": 0.01556456113130081, "run_count": 10, "total_questions": 1961, "avg_cost_per_task_usd": 0.18924138296787354, "avg_duration_per_task_ms": 516166.16930137685, "avg_output_tokens_per_task": 12560.821009688934, "last_run_timestamp": "2026-10-03T08:12:32.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9095505370786516, "accuracy_std_dev": 0.004402180380080482, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.02965778258426966, "avg_duration_per_task_ms": 474993.9230337079, "avg_output_tokens_per_task": 30670.006179775282, "last_run_timestamp": "2026-10-03T17:37:28.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9094274, "accuracy_std_dev": 0.002474202384608023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07117444141414142, "avg_duration_per_task_ms": 135218.17474747475, "avg_output_tokens_per_task": 27695.128282828282, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0054125, "avg_duration_per_task_ms": 235839.91414141413, "avg_output_tokens_per_task": 10749.717171717171, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9075092967032967, "accuracy_std_dev": 0.0008110000000000062, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.15048144505494504, "avg_duration_per_task_ms": 345399.21428571426, "avg_output_tokens_per_task": 38131.52197802198, "last_run_timestamp": "2026-10-03T10:04:03.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9074822295247725, "accuracy_std_dev": 0.016471532022249782, "run_count": 10, "total_questions": 1978, "avg_cost_per_task_usd": 0.03557361324570273, "avg_duration_per_task_ms": 322803.7876643074, "avg_output_tokens_per_task": 29631.699696663298, "last_run_timestamp": "2026-10-03T19:03:35.841Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.907407, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11648837373737374, "avg_duration_per_task_ms": 581221.1060606061, "avg_output_tokens_per_task": 46418.58585858586, "last_run_timestamp": "2026-10-03T09:31:25.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9072541818181818, "accuracy_std_dev": 0.02180228378286919, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03718532782369146, "avg_duration_per_task_ms": 268057.74655647384, "avg_output_tokens_per_task": 31182.948576675848, "last_run_timestamp": "2026-10-03T16:39:40.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.906355125, "accuracy_std_dev": 0.018459945425417037, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13796338068181818, "avg_duration_per_task_ms": 561092.216540404, "avg_output_tokens_per_task": 39407.719696969696, "last_run_timestamp": "2026-10-03T09:49:39.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9060606, "accuracy_std_dev": 0.006060466668500054, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.023695626262626262, "avg_duration_per_task_ms": 214413.58686868686, "avg_output_tokens_per_task": 29199.484848484848, "last_run_timestamp": "2026-10-03T09:22:46.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.90572375, "accuracy_std_dev": 0.009744176680843784, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03296931691919192, "avg_duration_per_task_ms": 228601.25883838383, "avg_output_tokens_per_task": 27296.915404040403, "last_run_timestamp": "2026-10-03T10:17:41.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9053872, "accuracy_std_dev": 0.006933048201188252, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.035172772727272726, "avg_duration_per_task_ms": 190581.2101010101, "avg_output_tokens_per_task": 29132.872727272726, "last_run_timestamp": "2026-10-03T09:10:43.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.9053669934810953, "accuracy_std_dev": 0.014124276768541987, "run_count": 8, "total_questions": 1534, "avg_cost_per_task_usd": 0.012634106910039114, "avg_duration_per_task_ms": 286058.7731421121, "avg_output_tokens_per_task": 17810.844198174706, "last_run_timestamp": "2026-10-03T10:17:33.683Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9041413987901349, "accuracy_std_dev": 0.012307350103169372, "run_count": 11, "total_questions": 2149, "avg_cost_per_task_usd": 0.02970322568636575, "avg_duration_per_task_ms": 256023.29827826895, "avg_output_tokens_per_task": 24623.136342484875, "last_run_timestamp": "2026-10-03T16:51:44.428Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9040978022813688, "accuracy_std_dev": 0.025559857939936256, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.040857811998310095, "avg_duration_per_task_ms": 257266.3920574567, "avg_output_tokens_per_task": 30699.54752851711, "last_run_timestamp": "2026-10-03T16:57:18.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.903872, "accuracy_std_dev": 0.028683231540396564, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04186602929292929, "avg_duration_per_task_ms": 291657.99141414143, "avg_output_tokens_per_task": 32019.437373737375, "last_run_timestamp": "2026-10-03T16:15:35.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9034975865992416, "accuracy_std_dev": 0.019591261601512255, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.03484569616519174, "avg_duration_per_task_ms": 327613.8377581121, "avg_output_tokens_per_task": 28971.93510324484, "last_run_timestamp": "2026-10-03T18:02:12.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.903424887142055, "accuracy_std_dev": 0.056166981894488544, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.14229890510948906, "avg_duration_per_task_ms": 308136.8270634475, "avg_output_tokens_per_task": 10189.49241998877, "last_run_timestamp": "2026-10-03T10:42:35.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9031985, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.033975815656565655, "avg_duration_per_task_ms": 141490.74494949495, "avg_output_tokens_per_task": 28136.507575757576, "last_run_timestamp": "2026-10-03T09:38:38.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902918, "accuracy_std_dev": 0.0007933738084913016, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14672305387205387, "avg_duration_per_task_ms": 133620.78956228957, "avg_output_tokens_per_task": 9652.131313131313, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.9028655747851002, "accuracy_std_dev": 0.042913716403447925, "run_count": 9, "total_questions": 1745, "avg_cost_per_task_usd": 0.010968459025787967, "avg_duration_per_task_ms": 489384.2842406877, "avg_output_tokens_per_task": 20816.36446991404, "last_run_timestamp": "2026-10-03T09:16:27.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9024267704752275, "accuracy_std_dev": 0.008260579897319528, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.03290473609706774, "avg_duration_per_task_ms": 147318.88675429727, "avg_output_tokens_per_task": 29037.182002022244, "last_run_timestamp": "2026-10-03T09:38:12.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.9023569999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05417848484848485, "avg_duration_per_task_ms": 36580.51515151515, "avg_output_tokens_per_task": 2499.1969696969695, "last_run_timestamp": "2026-10-03T09:27:36.763Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9023565, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.032186345959595956, "avg_duration_per_task_ms": 777530.196969697, "avg_output_tokens_per_task": 36633.931818181816, "last_run_timestamp": "2026-10-03T10:59:16.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9021696666666666, "accuracy_std_dev": 0.023279077501768247, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15281446520763184, "avg_duration_per_task_ms": 592828.4741863075, "avg_output_tokens_per_task": 38634.81593714927, "last_run_timestamp": "2026-10-03T10:13:07.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9021161428571426, "accuracy_std_dev": 0.016357597294997403, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.027857989898989896, "avg_duration_per_task_ms": 208577.13636363635, "avg_output_tokens_per_task": 26073.319624819625, "last_run_timestamp": "2026-10-03T09:35:22.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9017359124999998, "accuracy_std_dev": 0.023311648922467388, "run_count": 8, "total_questions": 1440, "avg_cost_per_task_usd": 0.15431451527777776, "avg_duration_per_task_ms": 567983.5055555556, "avg_output_tokens_per_task": 37740.19097222222, "last_run_timestamp": "2026-10-03T09:09:53.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9016834000000001, "accuracy_std_dev": 0.04222344006165297, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1424910212121212, "avg_duration_per_task_ms": 211878.58636363636, "avg_output_tokens_per_task": 9531.086363636363, "last_run_timestamp": "2026-10-03T10:10:50.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.9016834, "accuracy_std_dev": 0.01216780761846602, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.028648670707070705, "avg_duration_per_task_ms": 147445.17373737373, "avg_output_tokens_per_task": 28162.69292929293, "last_run_timestamp": "2026-10-03T09:28:15.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9014598562605278, "accuracy_std_dev": 0.03309502797836693, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.10678652554744525, "avg_duration_per_task_ms": 569188.9988770353, "avg_output_tokens_per_task": 39083.0421111735, "last_run_timestamp": "2026-10-03T10:32:36.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9010467416413374, "accuracy_std_dev": 0.006919339835562336, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.017340083080040522, "avg_duration_per_task_ms": 440190.9321175279, "avg_output_tokens_per_task": 28500.98986828774, "last_run_timestamp": "2026-10-03T09:52:27.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.9007667777777778, "accuracy_std_dev": 0.005624838482180366, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.20504940852974188, "avg_duration_per_task_ms": 98780.56790123456, "avg_output_tokens_per_task": 7889.076038159372, "last_run_timestamp": "2026-10-03T09:47:38.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9004328214285713, "accuracy_std_dev": 0.005913167322978487, "run_count": 28, "total_questions": 5544, "avg_cost_per_task_usd": 0.21622311616161613, "avg_duration_per_task_ms": 94665.56475468975, "avg_output_tokens_per_task": 7881.3840187590185, "last_run_timestamp": "2026-10-03T10:10:31.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9001119999999999, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.053618132996633, "avg_duration_per_task_ms": 27125.141414141413, "avg_output_tokens_per_task": 2464.4410774410776, "last_run_timestamp": "2026-10-03T09:16:11.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9000926755555556, "accuracy_std_dev": 0.009531680514027292, "run_count": 11, "total_questions": 1800, "avg_cost_per_task_usd": 0.022115208888888887, "avg_duration_per_task_ms": 208945.33944444446, "avg_output_tokens_per_task": 18295.701666666668, "last_run_timestamp": "2026-10-03T16:51:49.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8999998999999999, "accuracy_std_dev": 0.005644066804175849, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.012073083333333333, "avg_duration_per_task_ms": 179513.8106060606, "avg_output_tokens_per_task": 26194.345454545455, "last_run_timestamp": "2026-10-03T09:29:36.040Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8999248888888889, "accuracy_std_dev": 0.006354339085213833, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15479229124579125, "avg_duration_per_task_ms": 130090.1857463524, "avg_output_tokens_per_task": 9547.675645342311, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8998314999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0343437398989899, "avg_duration_per_task_ms": 823628.1161616162, "avg_output_tokens_per_task": 39114.656565656565, "last_run_timestamp": "2026-10-03T11:02:12.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14109027272727273, "avg_duration_per_task_ms": 131303, "avg_output_tokens_per_task": 9278.422558922559, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0412280202020202, "avg_duration_per_task_ms": 141835.40572390571, "avg_output_tokens_per_task": 27211.210437710437, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8994104999999998, "accuracy_std_dev": 0.004817079950550935, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.22572103156565654, "avg_duration_per_task_ms": 108993.70707070707, "avg_output_tokens_per_task": 8821.869949494949, "last_run_timestamp": "2026-10-03T09:42:54.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8993387243502051, "accuracy_std_dev": 0.026386926357751014, "run_count": 8, "total_questions": 1462, "avg_cost_per_task_usd": 0.027840691518467853, "avg_duration_per_task_ms": 740625.9062927497, "avg_output_tokens_per_task": 24883.40560875513, "last_run_timestamp": "2026-09-12T16:17:57.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8992219655172418, "accuracy_std_dev": 0.0066569301119999745, "run_count": 29, "total_questions": 5742, "avg_cost_per_task_usd": 0.22289186381051904, "avg_duration_per_task_ms": 94799.24817136886, "avg_output_tokens_per_task": 8115.898293277603, "last_run_timestamp": "2026-10-03T10:16:05.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8991767777777778, "accuracy_std_dev": 0.005290828285875197, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.058499294612794614, "avg_duration_per_task_ms": 32080.961840628508, "avg_output_tokens_per_task": 2531.0488215488217, "last_run_timestamp": "2026-10-03T10:16:57.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.005188652155184398, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03249949747474747, "avg_duration_per_task_ms": 137621.66666666666, "avg_output_tokens_per_task": 26156.715909090908, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8980803802559414, "accuracy_std_dev": 0.010966029181066412, "run_count": 6, "total_questions": 1094, "avg_cost_per_task_usd": 0.006137671846435101, "avg_duration_per_task_ms": 419028.80987202923, "avg_output_tokens_per_task": 12654.702925045704, "last_run_timestamp": "2026-10-03T10:14:40.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8980545555555557, "accuracy_std_dev": 0.0033774129188909115, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.057495572951739626, "avg_duration_per_task_ms": 25176.90740740741, "avg_output_tokens_per_task": 2488.7480359147025, "last_run_timestamp": "2026-10-03T09:48:22.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.8978673333333335, "accuracy_std_dev": 0.005382399950042904, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.057627444444444446, "avg_duration_per_task_ms": 26162.272727272728, "avg_output_tokens_per_task": 2526.015993265993, "last_run_timestamp": "2026-10-03T10:06:47.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8977272500000001, "accuracy_std_dev": 0.008615029218609743, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19720289898989898, "avg_duration_per_task_ms": 95317.51073232324, "avg_output_tokens_per_task": 7684.227904040404, "last_run_timestamp": "2026-10-03T09:39:18.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.8975330208728654, "accuracy_std_dev": 0.015763870376592163, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.11425380961416824, "avg_duration_per_task_ms": 628521.0765338393, "avg_output_tokens_per_task": 36072.062618595824, "last_run_timestamp": "2026-10-03T10:20:39.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8974932222222222, "accuracy_std_dev": 0.012382476271622079, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.20160183445566776, "avg_duration_per_task_ms": 97321.57126823794, "avg_output_tokens_per_task": 7848.616722783389, "last_run_timestamp": "2026-10-03T09:45:23.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8968568964143425, "accuracy_std_dev": 0.005278733293840445, "run_count": 4, "total_questions": 753, "avg_cost_per_task_usd": 0.15754132403718457, "avg_duration_per_task_ms": 84099.57768924303, "avg_output_tokens_per_task": 9420.674634794157, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8967218206896553, "accuracy_std_dev": 0.007385261912078655, "run_count": 20, "total_questions": 3915, "avg_cost_per_task_usd": 0.12559540842911876, "avg_duration_per_task_ms": 57752.34942528736, "avg_output_tokens_per_task": 4741.332567049809, "last_run_timestamp": "2026-10-03T10:05:55.795Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8964932281368821, "accuracy_std_dev": 0.008729450995767121, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.01934689100126743, "avg_duration_per_task_ms": 616078.9974651458, "avg_output_tokens_per_task": 17884.557667934092, "last_run_timestamp": "2026-10-03T10:04:37.350Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8960090269814502, "accuracy_std_dev": 0.005525506754035197, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.01363839797639123, "avg_duration_per_task_ms": 405081.17875210795, "avg_output_tokens_per_task": 20090.210792580103, "last_run_timestamp": "2026-10-03T11:58:57.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8956340674536255, "accuracy_std_dev": 0.02132427030288766, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.13012930916245083, "avg_duration_per_task_ms": 507416.96458684653, "avg_output_tokens_per_task": 37964.30185497471, "last_run_timestamp": "2026-10-03T10:12:16.841Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8956227499999999, "accuracy_std_dev": 0.005791223203046117, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.20478679124579122, "avg_duration_per_task_ms": 99940.12121212122, "avg_output_tokens_per_task": 7979.755892255892, "last_run_timestamp": "2026-10-03T09:40:16.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8951176, "accuracy_std_dev": 0.00756834082477789, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.12048651767676768, "avg_duration_per_task_ms": 57579.538383838386, "avg_output_tokens_per_task": 4603.499494949495, "last_run_timestamp": "2026-10-03T09:25:15.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8950130708661417, "accuracy_std_dev": 0.00891334037763057, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.048151682414698166, "avg_duration_per_task_ms": 43476.24934383202, "avg_output_tokens_per_task": 3503.036745406824, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8948596495327101, "accuracy_std_dev": 0.0171199232602798, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.022189622274143304, "avg_duration_per_task_ms": 231984.90109034267, "avg_output_tokens_per_task": 26737.70015576324, "last_run_timestamp": "2026-10-03T09:42:10.726Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8945527678958785, "accuracy_std_dev": 0.010412875331697246, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.012522617136659437, "avg_duration_per_task_ms": 291164.3582791034, "avg_output_tokens_per_task": 27224.147143890095, "last_run_timestamp": "2026-10-03T09:48:09.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8942841547799696, "accuracy_std_dev": 0.008875212006481873, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.22388517248356093, "avg_duration_per_task_ms": 107507.08952959029, "avg_output_tokens_per_task": 8731.16894284269, "last_run_timestamp": "2026-10-03T09:18:11.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8941264444444444, "accuracy_std_dev": 0.009636502052221474, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05178651795735129, "avg_duration_per_task_ms": 45445.343434343435, "avg_output_tokens_per_task": 3671.5897867564536, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8939392, "accuracy_std_dev": 0.021664239228738207, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.08778542626262627, "avg_duration_per_task_ms": 210435.61212121212, "avg_output_tokens_per_task": 29520.061616161616, "last_run_timestamp": "2026-10-03T16:44:09.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.893939, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021992727272727275, "avg_duration_per_task_ms": 693845.0404040404, "avg_output_tokens_per_task": 46651.48484848485, "last_run_timestamp": "2026-10-01T08:47:48.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8938831666666668, "accuracy_std_dev": 0.007122347684499023, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.20402719309764314, "avg_duration_per_task_ms": 70642.39461279461, "avg_output_tokens_per_task": 7430.633333333333, "last_run_timestamp": "2026-10-03T10:09:58.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.89322933125, "accuracy_std_dev": 0.0012764999999999582, "run_count": 2, "total_questions": 320, "avg_cost_per_task_usd": 0.033160396875, "avg_duration_per_task_ms": 1020849.565625, "avg_output_tokens_per_task": 37888.675, "last_run_timestamp": "2026-10-03T11:52:13.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.8931597755794235, "accuracy_std_dev": 0.01575096391955302, "run_count": 9, "total_questions": 1769, "avg_cost_per_task_usd": 0.11365974788015829, "avg_duration_per_task_ms": 394066.868852459, "avg_output_tokens_per_task": 7770.012436404749, "last_run_timestamp": "2026-10-03T10:26:28.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.89267675, "accuracy_std_dev": 0.006979218773437292, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037834488636363635, "avg_duration_per_task_ms": 50710.07575757576, "avg_output_tokens_per_task": 3480.1502525252527, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.892548239520958, "accuracy_std_dev": 0.006915975724525221, "run_count": 3, "total_questions": 501, "avg_cost_per_task_usd": 0.013722602794411178, "avg_duration_per_task_ms": 355406.4910179641, "avg_output_tokens_per_task": 23503.43512974052, "last_run_timestamp": "2026-10-03T10:04:24.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8922556666666667, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05459258585858586, "avg_duration_per_task_ms": 34041.639730639734, "avg_output_tokens_per_task": 2518.429292929293, "last_run_timestamp": "2026-10-03T09:50:45.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8919671427077888, "accuracy_std_dev": 0.027075382802174704, "run_count": 11, "total_questions": 1913, "avg_cost_per_task_usd": 0.2187960825927862, "avg_duration_per_task_ms": 381468.48144276004, "avg_output_tokens_per_task": 43318.19968635651, "last_run_timestamp": "2026-10-03T17:02:39.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8909932500000002, "accuracy_std_dev": 0.010197164639619184, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00789814393939394, "avg_duration_per_task_ms": 268244.18434343435, "avg_output_tokens_per_task": 27958.415404040403, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.89087581459854, "accuracy_std_dev": 0.012535536331663226, "run_count": 14, "total_questions": 2740, "avg_cost_per_task_usd": 0.0037107835766423355, "avg_duration_per_task_ms": 283929.7711678832, "avg_output_tokens_per_task": 20325.415693430656, "last_run_timestamp": "2026-10-03T09:45:10.741Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8905723333333333, "accuracy_std_dev": 0.021339306148253502, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13216584343434343, "avg_duration_per_task_ms": 251067.54713804714, "avg_output_tokens_per_task": 43934.003367003366, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.890572, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12533664393939395, "avg_duration_per_task_ms": 808381.0631313132, "avg_output_tokens_per_task": 49868.60353535353, "last_run_timestamp": "2026-10-03T09:33:15.909Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8903411328976034, "accuracy_std_dev": 0.020961163458907063, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.1928877625272331, "avg_duration_per_task_ms": 1363104.034858388, "avg_output_tokens_per_task": 56628.1917211329, "last_run_timestamp": "2026-10-03T17:43:47.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.005722799567422281, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.19800758417508418, "avg_duration_per_task_ms": 96141.14141414141, "avg_output_tokens_per_task": 7695.558922558923, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8900110000000001, "accuracy_std_dev": 0.0055552329084086844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2164912121212121, "avg_duration_per_task_ms": 104660.01851851853, "avg_output_tokens_per_task": 8434.90404040404, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8898507142857144, "accuracy_std_dev": 0.013463734020527524, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.013538019841269842, "avg_duration_per_task_ms": 210147.86976911977, "avg_output_tokens_per_task": 24623.09126984127, "last_run_timestamp": "2026-10-03T09:29:34.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8898480650076179, "accuracy_std_dev": 0.00936748461292215, "run_count": 30, "total_questions": 5907, "avg_cost_per_task_usd": 0.13325874741831725, "avg_duration_per_task_ms": 59111.40138818351, "avg_output_tokens_per_task": 4767.174030810902, "last_run_timestamp": "2026-10-03T09:22:12.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8895201249999999, "accuracy_std_dev": 0.018150193638343783, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1383345025252525, "avg_duration_per_task_ms": 580496.523989899, "avg_output_tokens_per_task": 34909.79734848485, "last_run_timestamp": "2026-10-03T10:33:13.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.88945, "accuracy_std_dev": 0.00975207530050226, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.024394900673400673, "avg_duration_per_task_ms": 319196.3552188552, "avg_output_tokens_per_task": 27886, "last_run_timestamp": "2026-10-03T10:36:49.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8894080654205608, "accuracy_std_dev": 0.00941594536414648, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.21110783605919006, "avg_duration_per_task_ms": 101949.81853582554, "avg_output_tokens_per_task": 8216.18808411215, "last_run_timestamp": "2026-10-03T09:19:25.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8892253999999999, "accuracy_std_dev": 0.008511315206241623, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.24073908914141412, "avg_duration_per_task_ms": 84900.21515151515, "avg_output_tokens_per_task": 8956.617424242424, "last_run_timestamp": "2026-10-03T09:30:01.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8888886666666669, "accuracy_std_dev": 0.007653072992087717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008966420875420876, "avg_duration_per_task_ms": 148767.4797979798, "avg_output_tokens_per_task": 31547.74074074074, "last_run_timestamp": "2026-10-03T10:11:42.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8885382849631968, "accuracy_std_dev": 0.024250159666278505, "run_count": 5, "total_questions": 951, "avg_cost_per_task_usd": 0.1142291840168244, "avg_duration_per_task_ms": 715234.0883280757, "avg_output_tokens_per_task": 30845.972660357518, "last_run_timestamp": "2026-10-03T09:27:52.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8883276666666667, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008124902356902356, "avg_duration_per_task_ms": 491163.50673400675, "avg_output_tokens_per_task": 22629.957912457914, "last_run_timestamp": "2026-10-01T11:20:35.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8881577165991904, "accuracy_std_dev": 0.008283430245979028, "run_count": 10, "total_questions": 1976, "avg_cost_per_task_usd": 0.015942041497975706, "avg_duration_per_task_ms": 164141.9762145749, "avg_output_tokens_per_task": 26128.10374493927, "last_run_timestamp": "2026-10-03T13:59:16.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.8880469999999999, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11352663888888888, "avg_duration_per_task_ms": 879783.9520202021, "avg_output_tokens_per_task": 45200.48484848485, "last_run_timestamp": "2026-10-03T10:42:46.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8880445319148936, "accuracy_std_dev": 0.009804096960454846, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.20574746403242147, "avg_duration_per_task_ms": 100391.19402228977, "avg_output_tokens_per_task": 8003.147416413374, "last_run_timestamp": "2026-10-03T09:42:47.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.887854866328257, "accuracy_std_dev": 0.036201773425286826, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.13917155047941343, "avg_duration_per_task_ms": 799755.2588832488, "avg_output_tokens_per_task": 34841.37958262832, "last_run_timestamp": "2026-10-03T10:47:57.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8877663333333334, "accuracy_std_dev": 0.005555502277522277, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03440870875420875, "avg_duration_per_task_ms": 50195.98148148148, "avg_output_tokens_per_task": 3309.2491582491584, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8867842499999999, "accuracy_std_dev": 0.003834422340783562, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.027946079545454545, "avg_duration_per_task_ms": 154476.702020202, "avg_output_tokens_per_task": 23124.396464646463, "last_run_timestamp": "2026-10-03T10:53:39.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8867689502093997, "accuracy_std_dev": 0.008568113097385388, "run_count": 11, "total_questions": 2149, "avg_cost_per_task_usd": 0.09704398464402049, "avg_duration_per_task_ms": 550284.3764541647, "avg_output_tokens_per_task": 29436.060027919964, "last_run_timestamp": "2026-10-03T17:04:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.031905555555555555, "avg_duration_per_task_ms": 767151.1060606061, "avg_output_tokens_per_task": 36810.59595959596, "last_run_timestamp": "2026-10-03T10:37:19.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8860007604485219, "accuracy_std_dev": 0.0277911335645022, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.04559343119266055, "avg_duration_per_task_ms": 698776.9031600407, "avg_output_tokens_per_task": 26722.172273190623, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8855853621621622, "accuracy_std_dev": 0.025702889954244423, "run_count": 3, "total_questions": 555, "avg_cost_per_task_usd": 0.012490181981981983, "avg_duration_per_task_ms": 346127.65405405406, "avg_output_tokens_per_task": 22279.807207207206, "last_run_timestamp": "2026-10-03T10:21:30.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8855220000000001, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1415003787878788, "avg_duration_per_task_ms": 184472.2676767677, "avg_output_tokens_per_task": 47043.42424242424, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8853753412384717, "accuracy_std_dev": 0.014100336545185728, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.06779363899868249, "avg_duration_per_task_ms": 300937.79380764166, "avg_output_tokens_per_task": 18734.31488801054, "last_run_timestamp": "2026-10-03T18:59:42.075Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8853190481927712, "accuracy_std_dev": 0.005795847495189989, "run_count": 4, "total_questions": 747, "avg_cost_per_task_usd": 0.0097919718875502, "avg_duration_per_task_ms": 50542.31994645247, "avg_output_tokens_per_task": 7272.5394912985275, "last_run_timestamp": "2026-10-01T19:09:12.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8846155538461539, "accuracy_std_dev": 0.0024989999999999735, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.10645926410256411, "avg_duration_per_task_ms": 514880.74102564104, "avg_output_tokens_per_task": 34685.94102564103, "last_run_timestamp": "2026-10-03T17:17:05.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8845953462138955, "accuracy_std_dev": 0.012778996436692273, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.007367392271662762, "avg_duration_per_task_ms": 467182.3048399688, "avg_output_tokens_per_task": 26353.96487119438, "last_run_timestamp": "2026-10-03T09:22:28.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8845707161572053, "accuracy_std_dev": 0.02143221713073307, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.16085208820960697, "avg_duration_per_task_ms": 756655.7283842795, "avg_output_tokens_per_task": 40813.194759825325, "last_run_timestamp": "2026-09-16T00:37:42.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.884269595505618, "accuracy_std_dev": 0.010583520080008851, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.018860816853932583, "avg_duration_per_task_ms": 74171.62584269662, "avg_output_tokens_per_task": 7862.741573033708, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.884259, "accuracy_std_dev": 0.003834285135458768, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028357917929292934, "avg_duration_per_task_ms": 90698.91287878787, "avg_output_tokens_per_task": 23491.551767676767, "last_run_timestamp": "2026-10-03T10:07:51.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8841843306451613, "accuracy_std_dev": 0.016935757620711884, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.04011120698924731, "avg_duration_per_task_ms": 259031.17204301077, "avg_output_tokens_per_task": 34712.397849462366, "last_run_timestamp": "2026-10-03T11:48:18.674Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8840485, "accuracy_std_dev": 0.014246189262395764, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.18107499936868682, "avg_duration_per_task_ms": 298705.5252525252, "avg_output_tokens_per_task": 45444.8851010101, "last_run_timestamp": "2026-10-03T10:13:34.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8839911818181818, "accuracy_std_dev": 0.015783702865574983, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1157818631772268, "avg_duration_per_task_ms": 609822.4752066116, "avg_output_tokens_per_task": 30247.83930211203, "last_run_timestamp": "2026-10-03T16:57:22.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8838383333333335, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13180945791245793, "avg_duration_per_task_ms": 248970.3754208754, "avg_output_tokens_per_task": 43817.097643097644, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8832300830755635, "accuracy_std_dev": 0.01311356273493304, "run_count": 23, "total_questions": 4526, "avg_cost_per_task_usd": 0.1913586323464428, "avg_duration_per_task_ms": 377955.7154220062, "avg_output_tokens_per_task": 31161.343570481662, "last_run_timestamp": "2026-10-03T16:59:23.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8829965000000001, "accuracy_std_dev": 0.009135988652028878, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.02256884242424242, "avg_duration_per_task_ms": 361325.05656565656, "avg_output_tokens_per_task": 25803.312121212122, "last_run_timestamp": "2026-10-03T11:18:47.195Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.882652041571654, "accuracy_std_dev": 0.010061769676464596, "run_count": 30, "total_questions": 5701, "avg_cost_per_task_usd": 0.13625698438870373, "avg_duration_per_task_ms": 62308.83844939484, "avg_output_tokens_per_task": 4947.5786704087, "last_run_timestamp": "2026-10-03T09:33:58.131Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.882236963045085, "accuracy_std_dev": 0.03276554586072114, "run_count": 7, "total_questions": 1353, "avg_cost_per_task_usd": 0.006029221729490022, "avg_duration_per_task_ms": 323803.38359201775, "avg_output_tokens_per_task": 25311.461197339246, "last_run_timestamp": "2026-10-03T09:49:39.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8816051722365038, "accuracy_std_dev": 0.07456878886603124, "run_count": 13, "total_questions": 2334, "avg_cost_per_task_usd": 0.010089964010282775, "avg_duration_per_task_ms": 149337.25192802056, "avg_output_tokens_per_task": 23612.06083976007, "last_run_timestamp": "2026-10-03T10:12:41.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8815301729019211, "accuracy_std_dev": 0.0059611119566738745, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.014240532861476238, "avg_duration_per_task_ms": 332365.4651162791, "avg_output_tokens_per_task": 26073.906976744187, "last_run_timestamp": "2026-10-03T10:46:54.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8815094743833018, "accuracy_std_dev": 0.009777163469023105, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.008091048070841239, "avg_duration_per_task_ms": 269562.41682479443, "avg_output_tokens_per_task": 28593.919038583175, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8813523333333333, "accuracy_std_dev": 0.012418655370047113, "run_count": 5, "total_questions": 774, "avg_cost_per_task_usd": 0.007396838501291989, "avg_duration_per_task_ms": 996564.0413436693, "avg_output_tokens_per_task": 29356.904392764856, "last_run_timestamp": "2026-10-03T09:53:30.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8813128999999998, "accuracy_std_dev": 0.018445584861695216, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.024727196464646464, "avg_duration_per_task_ms": 216333.7505050505, "avg_output_tokens_per_task": 25254.168181818182, "last_run_timestamp": "2026-10-03T09:13:55.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.880903752293578, "accuracy_std_dev": 0.010625848964200456, "run_count": 10, "total_questions": 1962, "avg_cost_per_task_usd": 0.005325050458715596, "avg_duration_per_task_ms": 400241.4791029562, "avg_output_tokens_per_task": 18619.075433231395, "last_run_timestamp": "2026-10-03T10:33:52.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8807518333333332, "accuracy_std_dev": 0.008440691666300569, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015967906565656567, "avg_duration_per_task_ms": 289503.96885521885, "avg_output_tokens_per_task": 24394.69781144781, "last_run_timestamp": "2026-10-03T09:36:17.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8804712499999999, "accuracy_std_dev": 0.006838655510222777, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028478232323232322, "avg_duration_per_task_ms": 124855.71338383839, "avg_output_tokens_per_task": 23539.155303030304, "last_run_timestamp": "2026-10-03T10:18:21.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.880471, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006361709595959596, "avg_duration_per_task_ms": 516019.3207070707, "avg_output_tokens_per_task": 34951.01515151515, "last_run_timestamp": "2026-10-03T11:41:19.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.880471, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07564930808080808, "avg_duration_per_task_ms": 291613.64898989897, "avg_output_tokens_per_task": 20887.656565656565, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8798590909090911, "accuracy_std_dev": 0.03263267695105222, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.14463881542699725, "avg_duration_per_task_ms": 346447.52938475664, "avg_output_tokens_per_task": 33431.59090909091, "last_run_timestamp": "2026-10-03T17:43:07.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8796295, "accuracy_std_dev": 0.015267782951365261, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028045234848484847, "avg_duration_per_task_ms": 222260.06818181818, "avg_output_tokens_per_task": 27117.741161616163, "last_run_timestamp": "2026-10-03T09:35:14.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8794991295774648, "accuracy_std_dev": 0.01942029328961192, "run_count": 11, "total_questions": 2130, "avg_cost_per_task_usd": 0.12990812910798122, "avg_duration_per_task_ms": 63213.040375586854, "avg_output_tokens_per_task": 4970.991549295774, "last_run_timestamp": "2026-10-03T10:06:49.792Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.879189, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.020723571428571428, "avg_duration_per_task_ms": 141095.54497354498, "avg_output_tokens_per_task": 17734.21164021164, "last_run_timestamp": "2026-09-15T00:33:26.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8790546552363299, "accuracy_std_dev": 0.030844451113161732, "run_count": 6, "total_questions": 1079, "avg_cost_per_task_usd": 0.1346450278035218, "avg_duration_per_task_ms": 284974.0537534754, "avg_output_tokens_per_task": 31126.302131603337, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8790282857142856, "accuracy_std_dev": 0.014040008533923795, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005288715728715729, "avg_duration_per_task_ms": 276008.51948051946, "avg_output_tokens_per_task": 28469.592352092353, "last_run_timestamp": "2026-10-03T17:07:12.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8787876666666666, "accuracy_std_dev": 0.005991714074916775, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007907055555555554, "avg_duration_per_task_ms": 79561.398989899, "avg_output_tokens_per_task": 7834.653198653199, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8787876363636363, "accuracy_std_dev": 0.006091232690182722, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.005402353535353536, "avg_duration_per_task_ms": 380915.31404958677, "avg_output_tokens_per_task": 29413.866391184572, "last_run_timestamp": "2026-10-03T09:51:11.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.8787874999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04341706060606061, "avg_duration_per_task_ms": 244250.4595959596, "avg_output_tokens_per_task": 37643.704545454544, "last_run_timestamp": "2026-10-03T10:51:03.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.876843640328698, "accuracy_std_dev": 0.014698550138959288, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.08638480594184576, "avg_duration_per_task_ms": 222833.85398230088, "avg_output_tokens_per_task": 21940.641592920354, "last_run_timestamp": "2026-10-03T09:40:59.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.8762625, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008190393939393938, "avg_duration_per_task_ms": 699109.8106060605, "avg_output_tokens_per_task": 28884.636363636364, "last_run_timestamp": "2026-07-18T08:31:23.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.8762625, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15284478282828282, "avg_duration_per_task_ms": 339007.8914141414, "avg_output_tokens_per_task": 34511.28535353536, "last_run_timestamp": "2026-07-18T08:26:40.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8761422857142855, "accuracy_std_dev": 0.023342020751758195, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.08516392496392496, "avg_duration_per_task_ms": 500065.10461760464, "avg_output_tokens_per_task": 35027.99062049062, "last_run_timestamp": "2026-10-03T17:33:23.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.8757659457720588, "accuracy_std_dev": 0.09251418420879137, "run_count": 11, "total_questions": 2176, "avg_cost_per_task_usd": 0.2399917977941176, "avg_duration_per_task_ms": 482095.9535845588, "avg_output_tokens_per_task": 59848.47564338235, "last_run_timestamp": "2026-10-03T10:18:13.963Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8756311250000001, "accuracy_std_dev": 0.007549310638023532, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04973767171717172, "avg_duration_per_task_ms": 283165.73863636365, "avg_output_tokens_per_task": 21128.222222222223, "last_run_timestamp": "2026-10-03T17:26:10.527Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8754206666666666, "accuracy_std_dev": 0.00901385812081723, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.061844909090909096, "avg_duration_per_task_ms": 262173.4595959596, "avg_output_tokens_per_task": 22965.14478114478, "last_run_timestamp": "2026-09-26T09:19:06.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8753905728900255, "accuracy_std_dev": 0.012094409094012564, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.035846069053708435, "avg_duration_per_task_ms": 251997.9505541347, "avg_output_tokens_per_task": 29905.09377664109, "last_run_timestamp": "2026-10-03T12:29:29.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8748596666666666, "accuracy_std_dev": 0.03480145848157005, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10809361952861953, "avg_duration_per_task_ms": 516538.27384960715, "avg_output_tokens_per_task": 34956.31930415264, "last_run_timestamp": "2026-10-03T09:34:05.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8747592697495183, "accuracy_std_dev": 0.04128530357388087, "run_count": 4, "total_questions": 519, "avg_cost_per_task_usd": 0.012099612716763006, "avg_duration_per_task_ms": 465229.36030828516, "avg_output_tokens_per_task": 26222.763005780347, "last_run_timestamp": "2026-10-03T09:28:13.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8746725555555557, "accuracy_std_dev": 0.018426695846040054, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13061521717171717, "avg_duration_per_task_ms": 63177.591470258136, "avg_output_tokens_per_task": 5005.783950617284, "last_run_timestamp": "2026-10-03T09:54:43.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8743494545454547, "accuracy_std_dev": 0.0150964078771537, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1106280968778696, "avg_duration_per_task_ms": 234771.5486685032, "avg_output_tokens_per_task": 31461.72359963269, "last_run_timestamp": "2026-10-03T16:40:49.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.874209816687737, "accuracy_std_dev": 0.005925415300582321, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.12289277560050568, "avg_duration_per_task_ms": 965329.4336283186, "avg_output_tokens_per_task": 43216.86472819216, "last_run_timestamp": "2026-10-03T17:24:45.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8741581249999999, "accuracy_std_dev": 0.004890233185582785, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07734060606060605, "avg_duration_per_task_ms": 324821.60984848486, "avg_output_tokens_per_task": 21989.39962121212, "last_run_timestamp": "2026-10-03T19:00:32.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.8740979999999999, "accuracy_std_dev": 0.06811118283890162, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.12654391486291486, "avg_duration_per_task_ms": 1351779.769119769, "avg_output_tokens_per_task": 56150.230158730155, "last_run_timestamp": "2026-10-03T17:53:34.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8736602893401014, "accuracy_std_dev": 0.018240759785588822, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.033190517766497465, "avg_duration_per_task_ms": 731117.2707275804, "avg_output_tokens_per_task": 38434.73942470389, "last_run_timestamp": "2026-10-03T11:12:58.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.87288105, "accuracy_std_dev": 0.008961225878751214, "run_count": 6, "total_questions": 1180, "avg_cost_per_task_usd": 0.11474949322033899, "avg_duration_per_task_ms": 657555.8720338983, "avg_output_tokens_per_task": 32804.78050847458, "last_run_timestamp": "2026-10-03T16:53:50.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.8727692590673575, "accuracy_std_dev": 0.006680537619749563, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.10616132815198619, "avg_duration_per_task_ms": 1070108.6269430052, "avg_output_tokens_per_task": 42555.205526770296, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8727401757049892, "accuracy_std_dev": 0.012263522167761748, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.00852685032537961, "avg_duration_per_task_ms": 434756.32754880696, "avg_output_tokens_per_task": 21739.99276934201, "last_run_timestamp": "2026-10-03T10:06:09.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.8726002303664923, "accuracy_std_dev": 0.013852891908270337, "run_count": 3, "total_questions": 573, "avg_cost_per_task_usd": 0.10812300000000001, "avg_duration_per_task_ms": 1136520.2198952879, "avg_output_tokens_per_task": 43220.56020942408, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8725159619400473, "accuracy_std_dev": 0.0310416281682088, "run_count": 15, "total_questions": 2969, "avg_cost_per_task_usd": 0.14988813371505558, "avg_duration_per_task_ms": 412802.2492421691, "avg_output_tokens_per_task": 11323.438531492086, "last_run_timestamp": "2026-10-03T09:38:56.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8724745, "accuracy_std_dev": 0.008858340547190563, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008325782828282828, "avg_duration_per_task_ms": 77307.88005050505, "avg_output_tokens_per_task": 7842.704545454545, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.872409485103627, "accuracy_std_dev": 0.04752937711760483, "run_count": 8, "total_questions": 1544, "avg_cost_per_task_usd": 0.056010884067357505, "avg_duration_per_task_ms": 651555.3283678757, "avg_output_tokens_per_task": 19822.61658031088, "last_run_timestamp": "2026-10-03T17:12:28.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8722849626411816, "accuracy_std_dev": 0.03667207040635816, "run_count": 6, "total_questions": 1151, "avg_cost_per_task_usd": 0.01310653171155517, "avg_duration_per_task_ms": 596394.4969591659, "avg_output_tokens_per_task": 34928.014769765425, "last_run_timestamp": "2026-10-03T09:08:44.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8722010494296577, "accuracy_std_dev": 0.0020781669055925365, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.045619747782002534, "avg_duration_per_task_ms": 189717.51711026617, "avg_output_tokens_per_task": 39672.14828897338, "last_run_timestamp": "2026-10-03T10:44:50.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8720537142857142, "accuracy_std_dev": 0.014113751204453896, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.22260263059163057, "avg_duration_per_task_ms": 1170078.2395382395, "avg_output_tokens_per_task": 55880.77417027417, "last_run_timestamp": "2026-10-03T16:37:45.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8718853999999999, "accuracy_std_dev": 0.004467018719459339, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06354945656565657, "avg_duration_per_task_ms": 279330.9616161616, "avg_output_tokens_per_task": 21096.323232323233, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8715861506361324, "accuracy_std_dev": 0.009632287238761115, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.008127365394402036, "avg_duration_per_task_ms": 324970.47837150126, "avg_output_tokens_per_task": 28742.415267175573, "last_run_timestamp": "2026-10-03T09:52:27.516Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8713843774220725, "accuracy_std_dev": 0.007143278612327366, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.07020013732097725, "avg_duration_per_task_ms": 276688.6663858467, "avg_output_tokens_per_task": 11428.55096882898, "last_run_timestamp": "2026-10-03T10:04:35.741Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8710750330630069, "accuracy_std_dev": 0.028110456226890915, "run_count": 9, "total_questions": 1603, "avg_cost_per_task_usd": 0.09870775982532752, "avg_duration_per_task_ms": 221158.11790393014, "avg_output_tokens_per_task": 26829.437305053027, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8710353744745891, "accuracy_std_dev": 0.16260220955169502, "run_count": 15, "total_questions": 2617, "avg_cost_per_task_usd": 0.142784888803974, "avg_duration_per_task_ms": 265667.35957202903, "avg_output_tokens_per_task": 10833.450133740926, "last_run_timestamp": "2026-10-03T10:04:25.516Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.87037025, "accuracy_std_dev": 0.013624975814198723, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.028775198232323235, "avg_duration_per_task_ms": 719468.7878787878, "avg_output_tokens_per_task": 34862.15782828283, "last_run_timestamp": "2026-10-03T16:57:03.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.870085, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.11565565128205128, "avg_duration_per_task_ms": 2243159.2153846156, "avg_output_tokens_per_task": 32886.94358974359, "last_run_timestamp": "2026-07-11T08:42:46.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8699495000000002, "accuracy_std_dev": 0.018722910197135487, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.030484161616161617, "avg_duration_per_task_ms": 134273.03535353535, "avg_output_tokens_per_task": 21527.786616161615, "last_run_timestamp": "2026-10-03T12:27:06.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8696488571428571, "accuracy_std_dev": 0.005236127806706348, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008200564935064936, "avg_duration_per_task_ms": 437965.4329004329, "avg_output_tokens_per_task": 29027.78354978355, "last_run_timestamp": "2026-10-03T16:58:32.515Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8695323612368023, "accuracy_std_dev": 0.009903791381808026, "run_count": 8, "total_questions": 1326, "avg_cost_per_task_usd": 0.0508528371040724, "avg_duration_per_task_ms": 417388.269984917, "avg_output_tokens_per_task": 16859.24886877828, "last_run_timestamp": "2026-10-03T18:37:36.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.010411879221190886, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07784478367003367, "avg_duration_per_task_ms": 107880.84595959596, "avg_output_tokens_per_task": 12619.909932659933, "last_run_timestamp": "2026-10-03T10:07:47.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8692988181818181, "accuracy_std_dev": 0.01574807678773497, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.008268114784205692, "avg_duration_per_task_ms": 218163.90817263545, "avg_output_tokens_per_task": 29305.03948576676, "last_run_timestamp": "2026-10-03T10:06:44.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.869276, "accuracy_std_dev": 0.008908612349855623, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.006512984848484849, "avg_duration_per_task_ms": 255242.29242424242, "avg_output_tokens_per_task": 18235.289393939394, "last_run_timestamp": "2026-10-03T09:02:06.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8692479999999999, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22171807575757574, "avg_duration_per_task_ms": 260359.50673400675, "avg_output_tokens_per_task": 22076.11111111111, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8692478333333333, "accuracy_std_dev": 0.012955574635611068, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.08235681228956229, "avg_duration_per_task_ms": 202641.79545454544, "avg_output_tokens_per_task": 13365.021885521886, "last_run_timestamp": "2026-10-03T09:43:24.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8691446346328195, "accuracy_std_dev": 0.025234963139792104, "run_count": 6, "total_questions": 1103, "avg_cost_per_task_usd": 0.01014118223028105, "avg_duration_per_task_ms": 374535.42611060746, "avg_output_tokens_per_task": 21668.27742520399, "last_run_timestamp": "2026-10-03T09:34:29.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8691187682758621, "accuracy_std_dev": 0.026187627614316582, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.1136188703448276, "avg_duration_per_task_ms": 769491.6009195403, "avg_output_tokens_per_task": 30218.06206896552, "last_run_timestamp": "2026-10-03T16:47:49.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8691185958620689, "accuracy_std_dev": 0.09871592059712983, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.026795262528735632, "avg_duration_per_task_ms": 215775.31816091953, "avg_output_tokens_per_task": 23251.332413793105, "last_run_timestamp": "2026-10-03T17:06:35.452Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13888562626262627, "avg_duration_per_task_ms": 1028019.2020202021, "avg_output_tokens_per_task": 46184.969696969696, "last_run_timestamp": "2026-07-18T16:16:27.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.8686868571428572, "accuracy_std_dev": 0.007365644001105447, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.025638126262626258, "avg_duration_per_task_ms": 240783.4256854257, "avg_output_tokens_per_task": 21122.886002886004, "last_run_timestamp": "2026-10-01T16:07:33.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8684572837465565, "accuracy_std_dev": 0.006350273025626523, "run_count": 4, "total_questions": 726, "avg_cost_per_task_usd": 0.0059327286501377405, "avg_duration_per_task_ms": 153900.47520661156, "avg_output_tokens_per_task": 21060.526170798898, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8679651428571429, "accuracy_std_dev": 0.004916957259716384, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07019697691197692, "avg_duration_per_task_ms": 226467.18253968254, "avg_output_tokens_per_task": 11432.188311688311, "last_run_timestamp": "2026-10-03T09:14:11.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8675644999999997, "accuracy_std_dev": 0.008112634749779732, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07749805892255893, "avg_duration_per_task_ms": 84869.17171717172, "avg_output_tokens_per_task": 12670.825757575758, "last_run_timestamp": "2026-10-03T09:34:51.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.86726020610687, "accuracy_std_dev": 0.0104972985066397, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.008138582061068702, "avg_duration_per_task_ms": 283436.84605597967, "avg_output_tokens_per_task": 30216.644402035625, "last_run_timestamp": "2026-10-03T17:03:44.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8669514, "accuracy_std_dev": 0.020262840823783834, "run_count": 6, "total_questions": 1170, "avg_cost_per_task_usd": 0.07919805042735042, "avg_duration_per_task_ms": 415100.8598290598, "avg_output_tokens_per_task": 23580.724786324787, "last_run_timestamp": "2026-10-03T09:45:55.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.866915421641791, "accuracy_std_dev": 0.02922832992755772, "run_count": 12, "total_questions": 2144, "avg_cost_per_task_usd": 0.010834175839552238, "avg_duration_per_task_ms": 328640.2140858209, "avg_output_tokens_per_task": 19735.994869402984, "last_run_timestamp": "2026-10-03T09:27:10.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8667625714285713, "accuracy_std_dev": 0.021717036478022374, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11692872871572872, "avg_duration_per_task_ms": 576009.277056277, "avg_output_tokens_per_task": 34814.431457431456, "last_run_timestamp": "2026-10-03T17:19:05.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8663944783673468, "accuracy_std_dev": 0.025544015942873897, "run_count": 7, "total_questions": 1225, "avg_cost_per_task_usd": 0.14541799102040817, "avg_duration_per_task_ms": 562916.0302040817, "avg_output_tokens_per_task": 51076.03020408163, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8663719999999998, "accuracy_std_dev": 0.0061821103395199835, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0037658851010101005, "avg_duration_per_task_ms": 453040.5378787879, "avg_output_tokens_per_task": 29702.50568181818, "last_run_timestamp": "2026-10-03T17:32:07.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8660411428571428, "accuracy_std_dev": 0.02151917954774678, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1960267453102453, "avg_duration_per_task_ms": 1120544.6767676768, "avg_output_tokens_per_task": 58159.38816738817, "last_run_timestamp": "2026-10-03T17:03:53.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8658246999999999, "accuracy_std_dev": 0.016649641828279688, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.08718337929292928, "avg_duration_per_task_ms": 202685.16717171718, "avg_output_tokens_per_task": 24084.222222222223, "last_run_timestamp": "2026-10-03T16:06:47.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8658007142857141, "accuracy_std_dev": 0.009679853093842238, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.06976018037518038, "avg_duration_per_task_ms": 223743.95815295816, "avg_output_tokens_per_task": 11357.300144300145, "last_run_timestamp": "2026-10-03T09:30:30.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.865536908952959, "accuracy_std_dev": 0.032953765579824115, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.10668413707637835, "avg_duration_per_task_ms": 700483.68892261, "avg_output_tokens_per_task": 37983.61153262519, "last_run_timestamp": "2026-10-03T16:56:36.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8654539693682195, "accuracy_std_dev": 0.014877891890633394, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.07507311359285258, "avg_duration_per_task_ms": 281173.1014677728, "avg_output_tokens_per_task": 20728.63433312061, "last_run_timestamp": "2026-10-03T17:17:14.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8653197142857143, "accuracy_std_dev": 0.01749528025100555, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.16135840259740258, "avg_duration_per_task_ms": 1133723.20995671, "avg_output_tokens_per_task": 54192.0101010101, "last_run_timestamp": "2026-10-03T17:43:48.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.8653196666666666, "accuracy_std_dev": 0.010378007269649058, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029422861952861954, "avg_duration_per_task_ms": 503984.98484848486, "avg_output_tokens_per_task": 34386.4898989899, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8651969388477632, "accuracy_std_dev": 0.0392226856144323, "run_count": 16, "total_questions": 3107, "avg_cost_per_task_usd": 0.004922328290955906, "avg_duration_per_task_ms": 103773.42162858062, "avg_output_tokens_per_task": 18562.228194399744, "last_run_timestamp": "2026-10-03T10:10:22.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8649764115566037, "accuracy_std_dev": 0.05470934774460394, "run_count": 5, "total_questions": 848, "avg_cost_per_task_usd": 0.08730664976415095, "avg_duration_per_task_ms": 645412.391509434, "avg_output_tokens_per_task": 26303.754716981133, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8649528761766837, "accuracy_std_dev": 0.01859749373173817, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.0065336495293265755, "avg_duration_per_task_ms": 348157.4265025344, "avg_output_tokens_per_task": 21785.366401158582, "last_run_timestamp": "2026-10-03T10:20:35.518Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.864636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.07161002030456853, "avg_duration_per_task_ms": 868248.1269035533, "avg_output_tokens_per_task": 28363.54314720812, "last_run_timestamp": "2026-10-03T14:58:52.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8642566289198608, "accuracy_std_dev": 0.016122613680445513, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.08601719425087108, "avg_duration_per_task_ms": 391195.4764808362, "avg_output_tokens_per_task": 24802.604529616725, "last_run_timestamp": "2026-10-03T09:33:56.518Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.864098116850127, "accuracy_std_dev": 0.012040853347389158, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.06984978577476715, "avg_duration_per_task_ms": 203374.89754445385, "avg_output_tokens_per_task": 24608.592718035565, "last_run_timestamp": "2026-10-03T09:19:06.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8639189175738725, "accuracy_std_dev": 0.02340391523335358, "run_count": 4, "total_questions": 643, "avg_cost_per_task_usd": 0.06298315552099534, "avg_duration_per_task_ms": 414941.54743390356, "avg_output_tokens_per_task": 21263.947122861588, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8638768571428571, "accuracy_std_dev": 0.019098135020606234, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1702009898989899, "avg_duration_per_task_ms": 1092527.1017316016, "avg_output_tokens_per_task": 59548.17532467532, "last_run_timestamp": "2026-10-03T17:43:01.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8636704174363595, "accuracy_std_dev": 0.014066506514437398, "run_count": 26, "total_questions": 4439, "avg_cost_per_task_usd": 0.1642218321694075, "avg_duration_per_task_ms": 249845.5370578959, "avg_output_tokens_per_task": 30059.050236539762, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8636361249999999, "accuracy_std_dev": 0.010205573274411139, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008622361111111112, "avg_duration_per_task_ms": 290441.0890151515, "avg_output_tokens_per_task": 30478.675505050505, "last_run_timestamp": "2026-10-03T17:35:32.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.863636, "accuracy_std_dev": 0.007273551310513057, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020974939393939393, "avg_duration_per_task_ms": 363755.6632996633, "avg_output_tokens_per_task": 34831.013468013465, "last_run_timestamp": "2026-10-03T11:53:46.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.863636, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22125699158249157, "avg_duration_per_task_ms": 259075.24915824915, "avg_output_tokens_per_task": 22030.511784511786, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.863005, "accuracy_std_dev": 0.016749196577746643, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07378729608585859, "avg_duration_per_task_ms": 283540.4873737374, "avg_output_tokens_per_task": 20998.69318181818, "last_run_timestamp": "2026-10-03T17:13:56.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.8628582711496745, "accuracy_std_dev": 0.021376331990005663, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.058844402024584234, "avg_duration_per_task_ms": 505421.9110629067, "avg_output_tokens_per_task": 33506.749096167754, "last_run_timestamp": "2026-10-03T17:23:31.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8625138888888889, "accuracy_std_dev": 0.02716878807440808, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00797921548821549, "avg_duration_per_task_ms": 504816.8608305275, "avg_output_tokens_per_task": 25080.10830527497, "last_run_timestamp": "2026-10-03T10:11:44.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8625137777777779, "accuracy_std_dev": 0.00880146315838422, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8542647558922559, "avg_duration_per_task_ms": 483293.1627384961, "avg_output_tokens_per_task": 33979.86139169472, "last_run_timestamp": "2026-10-03T09:32:40.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.8623800507614213, "accuracy_std_dev": 0.004695390245289045, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.04025210152284264, "avg_duration_per_task_ms": 352805.59390862944, "avg_output_tokens_per_task": 24224.402707275804, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8620937891696752, "accuracy_std_dev": 0.031861011712545156, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.12809203682310472, "avg_duration_per_task_ms": 701690.7270758123, "avg_output_tokens_per_task": 42897.71407942238, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8617657777777777, "accuracy_std_dev": 0.009167555942050057, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8453705723905725, "avg_duration_per_task_ms": 415699.1722783389, "avg_output_tokens_per_task": 33623.734006734005, "last_run_timestamp": "2026-10-03T09:39:37.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.861742375, "accuracy_std_dev": 0.01475920397529538, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.054362910353535354, "avg_duration_per_task_ms": 712745.5782828283, "avg_output_tokens_per_task": 33336.231691919194, "last_run_timestamp": "2026-10-03T17:03:53.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8616955969987624, "accuracy_std_dev": 0.021294415333698184, "run_count": 33, "total_questions": 6464, "avg_cost_per_task_usd": 0.2610163531868812, "avg_duration_per_task_ms": 338765.33369430696, "avg_output_tokens_per_task": 47192.89201732673, "last_run_timestamp": "2026-10-03T17:31:59.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8615786666666667, "accuracy_std_dev": 0.009382650075775203, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8094348821548821, "avg_duration_per_task_ms": 461988.8821548821, "avg_output_tokens_per_task": 32205.768237934906, "last_run_timestamp": "2026-10-03T10:13:16.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.015601910243620808, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.19432850925925926, "avg_duration_per_task_ms": 386703.5462962963, "avg_output_tokens_per_task": 56841.80892255892, "last_run_timestamp": "2026-10-03T16:58:25.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8611875454545455, "accuracy_std_dev": 0.05143184709419506, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.07813096464646466, "avg_duration_per_task_ms": 519431.6776859504, "avg_output_tokens_per_task": 37464.50964187328, "last_run_timestamp": "2026-10-03T17:22:47.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8611535819295559, "accuracy_std_dev": 0.018922304290795815, "run_count": 7, "total_questions": 1306, "avg_cost_per_task_usd": 0.005349909647779478, "avg_duration_per_task_ms": 768463.6661562021, "avg_output_tokens_per_task": 29577.52986217458, "last_run_timestamp": "2026-10-03T17:40:37.174Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8611110624999999, "accuracy_std_dev": 0.01746489113360268, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.11371201957070708, "avg_duration_per_task_ms": 408202.1710858586, "avg_output_tokens_per_task": 40614.27525252525, "last_run_timestamp": "2026-10-03T17:10:11.381Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.861111, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1977418813131313, "avg_duration_per_task_ms": 587463.0883838384, "avg_output_tokens_per_task": 58044.29797979798, "last_run_timestamp": "2026-07-18T08:26:48.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8610391240310077, "accuracy_std_dev": 0.010899356367337553, "run_count": 6, "total_questions": 1161, "avg_cost_per_task_usd": 0.05256056416881998, "avg_duration_per_task_ms": 426986.81912144704, "avg_output_tokens_per_task": 27212.199827734712, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8608923543307087, "accuracy_std_dev": 0.017439499999999997, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.003837814960629921, "avg_duration_per_task_ms": 298144.1771653543, "avg_output_tokens_per_task": 11523.807086614173, "last_run_timestamp": "2026-10-03T17:04:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8607369444444446, "accuracy_std_dev": 0.008186229884332315, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.8829101122334456, "avg_duration_per_task_ms": 416465.39506172837, "avg_output_tokens_per_task": 33444.42985409652, "last_run_timestamp": "2026-10-03T09:27:25.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.8602695000000001, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07283564141414141, "avg_duration_per_task_ms": 493206.3787878788, "avg_output_tokens_per_task": 56881.042929292926, "last_run_timestamp": "2026-10-03T17:41:27.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.8602692222222225, "accuracy_std_dev": 0.01392791126381983, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.7468858333333334, "avg_duration_per_task_ms": 493290.462962963, "avg_output_tokens_per_task": 29714.63973063973, "last_run_timestamp": "2026-10-03T09:34:14.060Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.860269, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07998095959595959, "avg_duration_per_task_ms": 537979.2878787878, "avg_output_tokens_per_task": 41976.78787878788, "last_run_timestamp": "2026-07-11T08:27:26.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8599929535603715, "accuracy_std_dev": 0.009491817962856216, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.1207670278637771, "avg_duration_per_task_ms": 352384.2745098039, "avg_output_tokens_per_task": 30417.953560371516, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.8599289999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 188, "avg_cost_per_task_usd": 0.023036143617021276, "avg_duration_per_task_ms": 1599986.2287234042, "avg_output_tokens_per_task": 29232.622340425532, "last_run_timestamp": "2026-07-18T08:57:54.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8598355714285714, "accuracy_std_dev": 0.011856733262862344, "run_count": 6, "total_questions": 1176, "avg_cost_per_task_usd": 0.14287056972789117, "avg_duration_per_task_ms": 268723.07568027213, "avg_output_tokens_per_task": 28901.64880952381, "last_run_timestamp": "2026-10-03T09:42:23.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8596616913043478, "accuracy_std_dev": 0.02724773313095075, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.10258134202898551, "avg_duration_per_task_ms": 343569.69492753624, "avg_output_tokens_per_task": 23077.884057971016, "last_run_timestamp": "2026-10-03T09:34:01.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "WandB", "accuracy": 0.8594275000000001, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005964128787878787, "avg_duration_per_task_ms": 438651.28282828286, "avg_output_tokens_per_task": 16769.555555555555, "last_run_timestamp": "2026-07-18T08:04:02.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8592168750000002, "accuracy_std_dev": 0.04194354929377548, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.17683030808080807, "avg_duration_per_task_ms": 45565.645833333336, "avg_output_tokens_per_task": 3337.2266414141413, "last_run_timestamp": "2026-10-03T09:18:12.856Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8591573843017328, "accuracy_std_dev": 0.02115249701193692, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.10401832415902142, "avg_duration_per_task_ms": 479500.45667686034, "avg_output_tokens_per_task": 30593.96636085627, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.8590881190053286, "accuracy_std_dev": 0.02052541308201858, "run_count": 4, "total_questions": 563, "avg_cost_per_task_usd": 0.08836215808170515, "avg_duration_per_task_ms": 517852.0088809947, "avg_output_tokens_per_task": 22138.96802841918, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8590668571428572, "accuracy_std_dev": 0.0075671013591461535, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.06653240404040404, "avg_duration_per_task_ms": 215829.40548340548, "avg_output_tokens_per_task": 10820.84126984127, "last_run_timestamp": "2026-10-03T10:11:33.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8583545954198474, "accuracy_std_dev": 0.04093739371296128, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.08452414249363867, "avg_duration_per_task_ms": 513092.07786259544, "avg_output_tokens_per_task": 22657.731297709925, "last_run_timestamp": "2026-10-03T17:07:22.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8583051666666667, "accuracy_std_dev": 0.011142254012192586, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006246021043771043, "avg_duration_per_task_ms": 610030.5193602693, "avg_output_tokens_per_task": 29029.87626262626, "last_run_timestamp": "2026-10-03T17:45:03.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8578642857142857, "accuracy_std_dev": 0.00981070615944329, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005024529581529581, "avg_duration_per_task_ms": 238361.04761904763, "avg_output_tokens_per_task": 28338.91414141414, "last_run_timestamp": "2026-10-03T16:57:52.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8578204545454546, "accuracy_std_dev": 0.010161020030789983, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8425653397612488, "avg_duration_per_task_ms": 451495.49678604223, "avg_output_tokens_per_task": 33526.43617998163, "last_run_timestamp": "2026-10-03T10:12:25.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8577779119999999, "accuracy_std_dev": 0.006748318809155334, "run_count": 4, "total_questions": 750, "avg_cost_per_task_usd": 0.23260101733333335, "avg_duration_per_task_ms": 433991.3, "avg_output_tokens_per_task": 51935.09333333333, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.8577439999999998, "accuracy_std_dev": 0.005891999999999953, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008277098484848486, "avg_duration_per_task_ms": 710306.3914141414, "avg_output_tokens_per_task": 29437.88888888889, "last_run_timestamp": "2026-07-18T08:30:06.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8576669736716133, "accuracy_std_dev": 0.03054982749271608, "run_count": 11, "total_questions": 2089, "avg_cost_per_task_usd": 0.025787480612733366, "avg_duration_per_task_ms": 476213.8056486357, "avg_output_tokens_per_task": 24583.015797032072, "last_run_timestamp": "2026-10-03T09:49:53.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8569513442340791, "accuracy_std_dev": 0.013334663992859697, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.10127325989672976, "avg_duration_per_task_ms": 639996.5054503729, "avg_output_tokens_per_task": 35441.2759609868, "last_run_timestamp": "2026-09-19T08:55:10.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8569023333333334, "accuracy_std_dev": 0.016379967131292483, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01228687542087542, "avg_duration_per_task_ms": 639472.5816498316, "avg_output_tokens_per_task": 33004.78787878788, "last_run_timestamp": "2026-10-03T09:19:24.461Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8568595349182764, "accuracy_std_dev": 0.021080058706304028, "run_count": 7, "total_questions": 1346, "avg_cost_per_task_usd": 0.0749669205052006, "avg_duration_per_task_ms": 467671.9881129272, "avg_output_tokens_per_task": 31991.333580980685, "last_run_timestamp": "2026-10-03T16:56:42.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8561403397129187, "accuracy_std_dev": 0.00800597163426111, "run_count": 16, "total_questions": 3135, "avg_cost_per_task_usd": 0.062250919936204144, "avg_duration_per_task_ms": 609156.9333333333, "avg_output_tokens_per_task": 34308.0692185008, "last_run_timestamp": "2026-10-03T17:02:38.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8550316666666666, "accuracy_std_dev": 0.01908702254878369, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8377942901234569, "avg_duration_per_task_ms": 451775.2300785634, "avg_output_tokens_per_task": 33341.289001122335, "last_run_timestamp": "2026-10-03T09:45:17.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8549221865284975, "accuracy_std_dev": 0.006241046279804878, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.02481670293609672, "avg_duration_per_task_ms": 142245.9913644214, "avg_output_tokens_per_task": 37982.34369602763, "last_run_timestamp": "2026-10-03T10:08:27.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8548752091836734, "accuracy_std_dev": 0.01338517551144632, "run_count": 6, "total_questions": 1176, "avg_cost_per_task_usd": 0.1090017406462585, "avg_duration_per_task_ms": 191170.07142857142, "avg_output_tokens_per_task": 28383.128401360544, "last_run_timestamp": "2026-10-03T09:07:14.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8548644736842105, "accuracy_std_dev": 0.024533883813741147, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.15887166959064328, "avg_duration_per_task_ms": 152796.97049441785, "avg_output_tokens_per_task": 15519.810738968634, "last_run_timestamp": "2026-10-03T09:27:53.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8547135999999999, "accuracy_std_dev": 0.03570492302526363, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1789077186868687, "avg_duration_per_task_ms": 44801.785858585856, "avg_output_tokens_per_task": 3372.5843434343433, "last_run_timestamp": "2026-10-03T09:14:11.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8546573333333333, "accuracy_std_dev": 0.015141104458467416, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007946281144781145, "avg_duration_per_task_ms": 637289.4747474748, "avg_output_tokens_per_task": 29143.16835016835, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8545491290566037, "accuracy_std_dev": 0.016485433040158266, "run_count": 21, "total_questions": 3975, "avg_cost_per_task_usd": 0.15811767823899373, "avg_duration_per_task_ms": 199794.28352201259, "avg_output_tokens_per_task": 28179.00402515723, "last_run_timestamp": "2026-10-03T09:59:27.517Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8545453999999999, "accuracy_std_dev": 0.04176428250119952, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.17536612575757574, "avg_duration_per_task_ms": 43901.6994949495, "avg_output_tokens_per_task": 3302.6388888888887, "last_run_timestamp": "2026-10-03T09:19:04.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8544685739130434, "accuracy_std_dev": 0.006277655854338451, "run_count": 14, "total_questions": 2760, "avg_cost_per_task_usd": 0.054625573913043464, "avg_duration_per_task_ms": 429842.79927536234, "avg_output_tokens_per_task": 27957.209420289855, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.854377, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.252522148989899, "avg_duration_per_task_ms": 295859.83838383836, "avg_output_tokens_per_task": 62942.86616161616, "last_run_timestamp": "2026-07-18T08:26:40.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8540964999999999, "accuracy_std_dev": 0.014793979437032262, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09252428872053871, "avg_duration_per_task_ms": 604262.7205387205, "avg_output_tokens_per_task": 27745.595117845118, "last_run_timestamp": "2026-10-03T09:59:46.432Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8537222222222223, "accuracy_std_dev": 0.017912548902175803, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.7545602300785634, "avg_duration_per_task_ms": 501662.9904601571, "avg_output_tokens_per_task": 30067.62962962963, "last_run_timestamp": "2026-10-03T09:46:07.607Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8535444426523295, "accuracy_std_dev": 0.03729714924788522, "run_count": 29, "total_questions": 5022, "avg_cost_per_task_usd": 0.012806513739546001, "avg_duration_per_task_ms": 408091.7891278375, "avg_output_tokens_per_task": 16584.42453205894, "last_run_timestamp": "2026-10-03T10:45:26.547Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8535352222222222, "accuracy_std_dev": 0.01802737382598276, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.8012126506734006, "avg_duration_per_task_ms": 478739.47053872055, "avg_output_tokens_per_task": 30372.589225589225, "last_run_timestamp": "2026-10-03T09:41:04.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8534097609942638, "accuracy_std_dev": 0.02277588037678235, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.04192918291905673, "avg_duration_per_task_ms": 666623.9623964309, "avg_output_tokens_per_task": 35056.57106437221, "last_run_timestamp": "2026-10-03T16:51:59.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8533247499999999, "accuracy_std_dev": 0.03034305490861954, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.8320753535353536, "avg_duration_per_task_ms": 427272.283459596, "avg_output_tokens_per_task": 33208.628787878784, "last_run_timestamp": "2026-10-03T09:44:19.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8531986, "accuracy_std_dev": 0.030700851614246794, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.12264740303030303, "avg_duration_per_task_ms": 564764.8696969697, "avg_output_tokens_per_task": 32450.073737373736, "last_run_timestamp": "2026-10-03T16:54:37.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8531398217391304, "accuracy_std_dev": 0.028934812364545037, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.012572355072463768, "avg_duration_per_task_ms": 541534.9311594203, "avg_output_tokens_per_task": 33634.00579710145, "last_run_timestamp": "2026-10-03T09:08:08.594Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.853024801369863, "accuracy_std_dev": 0.02604945551361019, "run_count": 6, "total_questions": 1168, "avg_cost_per_task_usd": 0.13878068065068494, "avg_duration_per_task_ms": 378053.25171232875, "avg_output_tokens_per_task": 31592.83133561644, "last_run_timestamp": "2026-10-03T09:40:06.763Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8528984782608696, "accuracy_std_dev": 0.014334577107719047, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.018874540579710147, "avg_duration_per_task_ms": 358751.04855072463, "avg_output_tokens_per_task": 37538.637681159424, "last_run_timestamp": "2026-10-03T10:13:52.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8527699090909091, "accuracy_std_dev": 0.015435615118376213, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7526788682277317, "avg_duration_per_task_ms": 500624.6831955923, "avg_output_tokens_per_task": 30015.131313131315, "last_run_timestamp": "2026-10-03T09:32:02.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8522125357142856, "accuracy_std_dev": 0.025081070583327737, "run_count": 28, "total_questions": 5544, "avg_cost_per_task_usd": 0.1703613856421356, "avg_duration_per_task_ms": 157589.71464646465, "avg_output_tokens_per_task": 15942.862554112555, "last_run_timestamp": "2026-10-03T09:43:58.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.8518516666666667, "accuracy_std_dev": 0.011904378363535908, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025817747474747476, "avg_duration_per_task_ms": 277441.0841750842, "avg_output_tokens_per_task": 21342.33164983165, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8518308614084509, "accuracy_std_dev": 0.02345464140331345, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.006422030422535212, "avg_duration_per_task_ms": 229362.44, "avg_output_tokens_per_task": 12608.14704225352, "last_run_timestamp": "2026-10-03T13:30:57.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8515010833333332, "accuracy_std_dev": 0.020334357614057767, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09762173148148147, "avg_duration_per_task_ms": 616187.7521043771, "avg_output_tokens_per_task": 42780.512205387204, "last_run_timestamp": "2026-10-03T16:57:15.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.85145952259887, "accuracy_std_dev": 0.01855480961260448, "run_count": 5, "total_questions": 708, "avg_cost_per_task_usd": 0.003704364406779661, "avg_duration_per_task_ms": 411396.5324858757, "avg_output_tokens_per_task": 7770.766949152542, "last_run_timestamp": "2026-10-03T09:41:45.583Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.8513253096590909, "accuracy_std_dev": 0.016587499999999977, "run_count": 2, "total_questions": 352, "avg_cost_per_task_usd": 0.105763125, "avg_duration_per_task_ms": 418327.64204545453, "avg_output_tokens_per_task": 31556.65625, "last_run_timestamp": "2026-07-18T08:30:34.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8508898571428573, "accuracy_std_dev": 0.059458570338462385, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1082869906204906, "avg_duration_per_task_ms": 468673.2698412698, "avg_output_tokens_per_task": 27557.489898989897, "last_run_timestamp": "2026-10-03T17:44:09.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8507314900265956, "accuracy_std_dev": 0.014339170843684643, "run_count": 8, "total_questions": 1504, "avg_cost_per_task_usd": 0.018270000664893617, "avg_duration_per_task_ms": 282286.72473404254, "avg_output_tokens_per_task": 34448.984707446805, "last_run_timestamp": "2026-10-03T09:44:00.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8506987191867852, "accuracy_std_dev": 0.008019330452707517, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.07234735196950444, "avg_duration_per_task_ms": 223593.9834815756, "avg_output_tokens_per_task": 22507.26048284625, "last_run_timestamp": "2026-10-03T09:46:11.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8506616837545127, "accuracy_std_dev": 0.011680143920346185, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.01837306498194946, "avg_duration_per_task_ms": 455469.79205776175, "avg_output_tokens_per_task": 36513.78483754513, "last_run_timestamp": "2026-10-03T09:41:16.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.850575, "accuracy_std_dev": null, "run_count": 1, "total_questions": 145, "avg_cost_per_task_usd": 0.015425489655172412, "avg_duration_per_task_ms": 62324.744827586204, "avg_output_tokens_per_task": 15913.096551724138, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8504743636363636, "accuracy_std_dev": 0.010422752256244077, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.02813888934802571, "avg_duration_per_task_ms": 684580.927456382, "avg_output_tokens_per_task": 32088.465564738293, "last_run_timestamp": "2026-10-03T09:34:29.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8504488333333333, "accuracy_std_dev": 0.010441946137520948, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10296402020202021, "avg_duration_per_task_ms": 627829.3468013467, "avg_output_tokens_per_task": 32815.65572390572, "last_run_timestamp": "2026-10-03T17:01:05.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8503154279475983, "accuracy_std_dev": 0.03140494390885563, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.09674545997088793, "avg_duration_per_task_ms": 1349515.0895196507, "avg_output_tokens_per_task": 43276.89374090247, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8502597003792668, "accuracy_std_dev": 0.008291599964555819, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.05148469405815423, "avg_duration_per_task_ms": 290540.5630004214, "avg_output_tokens_per_task": 46134.28908554572, "last_run_timestamp": "2026-10-03T17:05:07.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8500861979274612, "accuracy_std_dev": 0.01857196094762209, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.016829795854922277, "avg_duration_per_task_ms": 389325.4870466321, "avg_output_tokens_per_task": 22513.535751295338, "last_run_timestamp": "2026-08-29T18:07:13.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8499464590032153, "accuracy_std_dev": 0.022950616654380467, "run_count": 7, "total_questions": 1244, "avg_cost_per_task_usd": 0.010361853697749194, "avg_duration_per_task_ms": 360749.26848874596, "avg_output_tokens_per_task": 20512.316720257233, "last_run_timestamp": "2026-10-03T09:28:36.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8499387272727273, "accuracy_std_dev": 0.010489521646870508, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.040005734159779614, "avg_duration_per_task_ms": 395404.1997245179, "avg_output_tokens_per_task": 40289.957300275484, "last_run_timestamp": "2026-10-03T19:06:53.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8498315000000001, "accuracy_std_dev": 0.03952326621940551, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.17816506616161618, "avg_duration_per_task_ms": 45707.15909090909, "avg_output_tokens_per_task": 3356.5939393939393, "last_run_timestamp": "2026-10-03T09:19:41.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8498314, "accuracy_std_dev": 0.02688129444874262, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.15721394040404038, "avg_duration_per_task_ms": 152201.82676767677, "avg_output_tokens_per_task": 15506.534343434343, "last_run_timestamp": "2026-10-03T09:23:24.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8492828305531167, "accuracy_std_dev": 0.030679553487986455, "run_count": 6, "total_questions": 1139, "avg_cost_per_task_usd": 0.1125236496927129, "avg_duration_per_task_ms": 549482.1018437225, "avg_output_tokens_per_task": 33342.05618964003, "last_run_timestamp": "2026-10-03T10:16:13.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8492264101265824, "accuracy_std_dev": 0.04905497536664576, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.1081006405063291, "avg_duration_per_task_ms": 612102.1265822785, "avg_output_tokens_per_task": 27354.124050632912, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8487653333333334, "accuracy_std_dev": 0.011791380929400174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01937495622895623, "avg_duration_per_task_ms": 442506.297979798, "avg_output_tokens_per_task": 38526.9595959596, "last_run_timestamp": "2026-10-03T09:49:43.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8486995349544073, "accuracy_std_dev": 0.009718197845279743, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.10196987639311045, "avg_duration_per_task_ms": 302885.430597771, "avg_output_tokens_per_task": 30304.676798378925, "last_run_timestamp": "2026-10-03T09:33:45.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.848597158186864, "accuracy_std_dev": 0.013754474089263532, "run_count": 6, "total_questions": 1081, "avg_cost_per_task_usd": 0.006980295097132284, "avg_duration_per_task_ms": 490407.8658649399, "avg_output_tokens_per_task": 17200.208140610546, "last_run_timestamp": "2026-10-03T16:56:50.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8484290068965519, "accuracy_std_dev": 0.010032386386676414, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.03036523402298851, "avg_duration_per_task_ms": 666184.1756321839, "avg_output_tokens_per_task": 31675.185287356322, "last_run_timestamp": "2026-10-03T10:03:21.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8482143392857143, "accuracy_std_dev": 0.04522948547653644, "run_count": 6, "total_questions": 1176, "avg_cost_per_task_usd": 0.17130176700680272, "avg_duration_per_task_ms": 985601.3954081633, "avg_output_tokens_per_task": 51250.37244897959, "last_run_timestamp": "2026-10-03T17:02:58.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8480139539435808, "accuracy_std_dev": 0.013068837655060239, "run_count": 9, "total_questions": 1737, "avg_cost_per_task_usd": 0.02671003742084053, "avg_duration_per_task_ms": 67728.42774899252, "avg_output_tokens_per_task": 19651.77605066206, "last_run_timestamp": "2026-10-03T09:55:20.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8480035714285714, "accuracy_std_dev": 0.037501057988340775, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.045334733044733046, "avg_duration_per_task_ms": 477671.12987012987, "avg_output_tokens_per_task": 32379.50144300144, "last_run_timestamp": "2026-10-03T17:33:25.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.8479612021276596, "accuracy_std_dev": 0.006925500000000029, "run_count": 2, "total_questions": 376, "avg_cost_per_task_usd": 0.07202128191489361, "avg_duration_per_task_ms": 1392719.3138297873, "avg_output_tokens_per_task": 36985.77127659575, "last_run_timestamp": "2026-07-18T16:22:39.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8478198075949368, "accuracy_std_dev": 0.015598435238474664, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.07492492320675105, "avg_duration_per_task_ms": 285938.56540084386, "avg_output_tokens_per_task": 19472.943459915612, "last_run_timestamp": "2026-10-03T10:12:41.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8478035581395349, "accuracy_std_dev": 0.061331422730297465, "run_count": 7, "total_questions": 1290, "avg_cost_per_task_usd": 0.010003304651162791, "avg_duration_per_task_ms": 534481.4193798449, "avg_output_tokens_per_task": 19807.151937984498, "last_run_timestamp": "2026-10-03T10:05:20.841Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.8476773606557377, "accuracy_std_dev": 0.0038120000000000376, "run_count": 2, "total_questions": 244, "avg_cost_per_task_usd": 0.014137975409836067, "avg_duration_per_task_ms": 314874.34016393445, "avg_output_tokens_per_task": 11748.155737704918, "last_run_timestamp": "2026-07-18T16:53:38.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8475308251851853, "accuracy_std_dev": 0.02209777163862587, "run_count": 14, "total_questions": 2700, "avg_cost_per_task_usd": 0.16495124851851853, "avg_duration_per_task_ms": 300213.63333333336, "avg_output_tokens_per_task": 41197.25444444444, "last_run_timestamp": "2026-10-03T17:08:51.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8474135454545455, "accuracy_std_dev": 0.02039593415527014, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12871944444444444, "avg_duration_per_task_ms": 433056.7759412305, "avg_output_tokens_per_task": 35508.42056932966, "last_run_timestamp": "2026-10-03T16:40:33.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8468523098859316, "accuracy_std_dev": 0.015292048274757053, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.03700323827629911, "avg_duration_per_task_ms": 627930.3586818757, "avg_output_tokens_per_task": 34553.87896070976, "last_run_timestamp": "2026-10-03T17:17:28.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8467494829721361, "accuracy_std_dev": 0.006308499999999995, "run_count": 2, "total_questions": 323, "avg_cost_per_task_usd": 0.009286786377708979, "avg_duration_per_task_ms": 438695.826625387, "avg_output_tokens_per_task": 23117.473684210527, "last_run_timestamp": "2026-10-03T09:32:36.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8466377271407838, "accuracy_std_dev": 0.016590356861005438, "run_count": 7, "total_questions": 1378, "avg_cost_per_task_usd": 0.009304855587808419, "avg_duration_per_task_ms": 444689.5050798258, "avg_output_tokens_per_task": 28194.9412191582, "last_run_timestamp": "2026-10-03T10:04:48.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8466045281660314, "accuracy_std_dev": 0.011422067985981737, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.1267483756882677, "avg_duration_per_task_ms": 355086.75984752225, "avg_output_tokens_per_task": 33407.23930537908, "last_run_timestamp": "2026-10-03T09:40:39.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8462925557680051, "accuracy_std_dev": 0.011842625025068353, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.006960335245379223, "avg_duration_per_task_ms": 537888.3499043977, "avg_output_tokens_per_task": 17232.114722753347, "last_run_timestamp": "2026-10-03T17:01:29.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8461756369863013, "accuracy_std_dev": 0.003462455005724478, "run_count": 3, "total_questions": 584, "avg_cost_per_task_usd": 0.07262447602739726, "avg_duration_per_task_ms": 181056.27226027398, "avg_output_tokens_per_task": 31634.909246575342, "last_run_timestamp": "2026-10-03T09:37:27.212Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8461699999999999, "accuracy_std_dev": 0.03781276874945815, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6916325820707071, "avg_duration_per_task_ms": 471363.1818181818, "avg_output_tokens_per_task": 27729.57702020202, "last_run_timestamp": "2026-10-03T09:31:25.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8456601854103344, "accuracy_std_dev": 0.00860173579226889, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.02046987537993921, "avg_duration_per_task_ms": 602884.1205673759, "avg_output_tokens_per_task": 23203.004052684904, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8454851888607595, "accuracy_std_dev": 0.015145407351405253, "run_count": 10, "total_questions": 1975, "avg_cost_per_task_usd": 0.08874519088607594, "avg_duration_per_task_ms": 411899.82784810127, "avg_output_tokens_per_task": 22073.469873417722, "last_run_timestamp": "2026-10-03T17:44:12.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8453581428571427, "accuracy_std_dev": 0.013659783740481936, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10694367099567102, "avg_duration_per_task_ms": 329051.74314574315, "avg_output_tokens_per_task": 29197.39393939394, "last_run_timestamp": "2026-10-03T09:28:28.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8453473636363636, "accuracy_std_dev": 0.010732273090354644, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.10142870936639119, "avg_duration_per_task_ms": 595868.2428833792, "avg_output_tokens_per_task": 32109.968778696053, "last_run_timestamp": "2026-10-03T18:05:42.597Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8453441605206073, "accuracy_std_dev": 0.024165885254643357, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.23573006531694388, "avg_duration_per_task_ms": 512868.35454326344, "avg_output_tokens_per_task": 44701.53217642805, "last_run_timestamp": "2026-10-03T16:58:25.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8450359687930242, "accuracy_std_dev": 0.011573586447164944, "run_count": 12, "total_questions": 2179, "avg_cost_per_task_usd": 0.015310958237723726, "avg_duration_per_task_ms": 581526.1060119321, "avg_output_tokens_per_task": 16391.44194584672, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8450058498194947, "accuracy_std_dev": 0.034613668932657105, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.17534774657039712, "avg_duration_per_task_ms": 1010638.444765343, "avg_output_tokens_per_task": 53792.70180505415, "last_run_timestamp": "2026-10-03T16:47:29.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8446367142857142, "accuracy_std_dev": 0.0059496991692085995, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.009103200577200578, "avg_duration_per_task_ms": 400631.7698412698, "avg_output_tokens_per_task": 18223.642135642134, "last_run_timestamp": "2026-10-03T17:22:47.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8444975263157894, "accuracy_std_dev": 0.05215908171451967, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.18362076209463052, "avg_duration_per_task_ms": 40742.116427432215, "avg_output_tokens_per_task": 3305.715842636895, "last_run_timestamp": "2026-10-03T09:56:46.597Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8444934569536424, "accuracy_std_dev": 0.014326304586390738, "run_count": 7, "total_questions": 1359, "avg_cost_per_task_usd": 0.07969532008830021, "avg_duration_per_task_ms": 899418.5172921265, "avg_output_tokens_per_task": 40442.24208977189, "last_run_timestamp": "2026-08-30T00:19:02.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8443601352718078, "accuracy_std_dev": 0.03374720321086705, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.054606860092709646, "avg_duration_per_task_ms": 224549.4858828487, "avg_output_tokens_per_task": 22651.43194268858, "last_run_timestamp": "2026-10-03T10:19:52.322Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8442346282818979, "accuracy_std_dev": 0.01874547476103309, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.2081649216950714, "avg_duration_per_task_ms": 311887.5845232612, "avg_output_tokens_per_task": 47232.94426531552, "last_run_timestamp": "2026-10-03T16:37:10.188Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8441076000000001, "accuracy_std_dev": 0.00842413538827578, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1428492090909091, "avg_duration_per_task_ms": 160872.74545454545, "avg_output_tokens_per_task": 32292.333333333332, "last_run_timestamp": "2026-10-03T09:56:48.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8441076, "accuracy_std_dev": 0.036707935445077826, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015406010101010103, "avg_duration_per_task_ms": 387191.47878787876, "avg_output_tokens_per_task": 27797.58787878788, "last_run_timestamp": "2026-10-03T16:57:48.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.8437161084142395, "accuracy_std_dev": 0.013475684773786068, "run_count": 7, "total_questions": 1236, "avg_cost_per_task_usd": 0.006951031553398057, "avg_duration_per_task_ms": 652065.4733009709, "avg_output_tokens_per_task": 17156.162621359224, "last_run_timestamp": "2026-10-03T17:20:08.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8435618989169676, "accuracy_std_dev": 0.03832760617920939, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.16082656317689534, "avg_duration_per_task_ms": 1412866.806498195, "avg_output_tokens_per_task": 51968.49458483754, "last_run_timestamp": "2026-10-03T17:37:27.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8434342499999999, "accuracy_std_dev": 0.008881818827667008, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.14890805050505052, "avg_duration_per_task_ms": 162687.35606060605, "avg_output_tokens_per_task": 32857.1712962963, "last_run_timestamp": "2026-10-03T10:18:11.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8423878991712708, "accuracy_std_dev": 0.05766154228422131, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.07142284576427256, "avg_duration_per_task_ms": 676306.011510129, "avg_output_tokens_per_task": 44005.55662983425, "last_run_timestamp": "2026-10-03T17:06:14.822Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.842382125, "accuracy_std_dev": 0.024293570854433386, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.06867556881313132, "avg_duration_per_task_ms": 555486.8421717172, "avg_output_tokens_per_task": 21945.48106060606, "last_run_timestamp": "2026-10-03T18:21:53.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8423120833333333, "accuracy_std_dev": 0.02273594124016837, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.06854938888888887, "avg_duration_per_task_ms": 673113.2765151515, "avg_output_tokens_per_task": 51666.62584175084, "last_run_timestamp": "2026-10-03T16:57:33.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8422381263537906, "accuracy_std_dev": 0.013606259020414556, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.1362164772563177, "avg_duration_per_task_ms": 189944.0108303249, "avg_output_tokens_per_task": 30733.362454873648, "last_run_timestamp": "2026-10-03T09:32:18.488Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.842151959435626, "accuracy_std_dev": 0.020139741016761404, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.02213832098765432, "avg_duration_per_task_ms": 283612.92063492065, "avg_output_tokens_per_task": 18387.772486772486, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8419798924812032, "accuracy_std_dev": 0.0161432408803948, "run_count": 7, "total_questions": 1330, "avg_cost_per_task_usd": 0.01795621127819549, "avg_duration_per_task_ms": 242875.14962406014, "avg_output_tokens_per_task": 35697.367669172934, "last_run_timestamp": "2026-10-03T09:24:57.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8417268575037897, "accuracy_std_dev": 0.03274733184905639, "run_count": 30, "total_questions": 5937, "avg_cost_per_task_usd": 0.16544357267980464, "avg_duration_per_task_ms": 149160.101061142, "avg_output_tokens_per_task": 15291.950816910898, "last_run_timestamp": "2026-10-03T10:45:13.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8413977096774194, "accuracy_std_dev": 0.007842951293996439, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.03688066532258065, "avg_duration_per_task_ms": 47685.283602150535, "avg_output_tokens_per_task": 14664.099462365592, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8411896666666667, "accuracy_std_dev": 0.008729468916008341, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.036658867003367, "avg_duration_per_task_ms": 51092.77104377104, "avg_output_tokens_per_task": 14574.808080808081, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8410581923809524, "accuracy_std_dev": 0.018771452424076705, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.04652745714285714, "avg_duration_per_task_ms": 641076.9504761904, "avg_output_tokens_per_task": 37228.5180952381, "last_run_timestamp": "2026-10-03T16:58:17.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8409091945454545, "accuracy_std_dev": 0.023300437613580863, "run_count": 8, "total_questions": 1100, "avg_cost_per_task_usd": 0.01265548181818182, "avg_duration_per_task_ms": 72073.20181818181, "avg_output_tokens_per_task": 11616.710909090909, "last_run_timestamp": "2026-10-03T17:24:07.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8409091666666667, "accuracy_std_dev": 0.012885450586050746, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12154702777777776, "avg_duration_per_task_ms": 341680.0117845118, "avg_output_tokens_per_task": 27429.664983164985, "last_run_timestamp": "2026-10-03T09:27:41.412Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8408324545454547, "accuracy_std_dev": 0.07268828033879719, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.18564792516069786, "avg_duration_per_task_ms": 528994.2309458219, "avg_output_tokens_per_task": 42221.45684113866, "last_run_timestamp": "2026-10-03T18:53:59.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.840817973713034, "accuracy_std_dev": 0.020791979459397293, "run_count": 5, "total_questions": 913, "avg_cost_per_task_usd": 0.1426375947426068, "avg_duration_per_task_ms": 229737.46549835705, "avg_output_tokens_per_task": 33714.36035049288, "last_run_timestamp": "2026-10-03T09:33:59.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.84048825, "accuracy_std_dev": 0.004667830993887862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06197197222222223, "avg_duration_per_task_ms": 84839.70707070707, "avg_output_tokens_per_task": 13643.420454545454, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.84048825, "accuracy_std_dev": 0.007080042491927587, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0629129558080808, "avg_duration_per_task_ms": 337433.2537878788, "avg_output_tokens_per_task": 20687.39898989899, "last_run_timestamp": "2026-10-03T14:07:19.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8404742875816994, "accuracy_std_dev": 0.009920155987499742, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.027662923747276687, "avg_duration_per_task_ms": 532773.5497458243, "avg_output_tokens_per_task": 22865.482207697893, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8402775531250001, "accuracy_std_dev": 0.007466846475587956, "run_count": 10, "total_questions": 1920, "avg_cost_per_task_usd": 0.039408469791666664, "avg_duration_per_task_ms": 51791.722395833334, "avg_output_tokens_per_task": 14757.431770833333, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.8400541048387097, "accuracy_std_dev": 0.0023684999999999956, "run_count": 2, "total_questions": 248, "avg_cost_per_task_usd": 0.016170879032258067, "avg_duration_per_task_ms": 368527.8064516129, "avg_output_tokens_per_task": 11745.584677419354, "last_run_timestamp": "2026-07-18T16:47:35.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8398692413632118, "accuracy_std_dev": 0.024400746937490225, "run_count": 11, "total_questions": 2142, "avg_cost_per_task_usd": 0.20761488702147526, "avg_duration_per_task_ms": 295169.1633986928, "avg_output_tokens_per_task": 47026.597572362276, "last_run_timestamp": "2026-10-03T16:57:30.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8391689163108454, "accuracy_std_dev": 0.01637841911378101, "run_count": 6, "total_questions": 1171, "avg_cost_per_task_usd": 0.014026462852263022, "avg_duration_per_task_ms": 450960.66951323656, "avg_output_tokens_per_task": 38615.76686592656, "last_run_timestamp": "2026-10-03T09:28:28.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8389739548751007, "accuracy_std_dev": 0.023840636777552385, "run_count": 7, "total_questions": 1241, "avg_cost_per_task_usd": 0.07291945044319098, "avg_duration_per_task_ms": 368316.54069298954, "avg_output_tokens_per_task": 24012.563255439163, "last_run_timestamp": "2026-10-03T16:56:41.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8389449999999999, "accuracy_std_dev": 0.010132054118818512, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.027894321548821547, "avg_duration_per_task_ms": 402197.4781144781, "avg_output_tokens_per_task": 14811.760942760942, "last_run_timestamp": "2026-10-03T10:19:34.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8388887961538462, "accuracy_std_dev": 0.016188823238869932, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.034323907051282056, "avg_duration_per_task_ms": 565277.3544871794, "avg_output_tokens_per_task": 32958.441666666666, "last_run_timestamp": "2026-10-03T16:57:14.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8388647142857143, "accuracy_std_dev": 0.020435361360112217, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02562200432900433, "avg_duration_per_task_ms": 254478.84992784992, "avg_output_tokens_per_task": 21304.302308802307, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8388431247933885, "accuracy_std_dev": 0.0097334078077013, "run_count": 7, "total_questions": 1210, "avg_cost_per_task_usd": 0.08136244214876033, "avg_duration_per_task_ms": 548386.6462809917, "avg_output_tokens_per_task": 20121.671074380167, "last_run_timestamp": "2026-10-01T16:08:32.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.8386665466666666, "accuracy_std_dev": 0.00007150000000000212, "run_count": 2, "total_questions": 375, "avg_cost_per_task_usd": 0.031960357333333335, "avg_duration_per_task_ms": 164702.43466666667, "avg_output_tokens_per_task": 26929.237333333334, "last_run_timestamp": "2026-07-18T08:31:24.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8384329244791666, "accuracy_std_dev": 0.01468840893357752, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.221448376953125, "avg_duration_per_task_ms": 429166.3111979167, "avg_output_tokens_per_task": 50182.490234375, "last_run_timestamp": "2026-10-03T16:59:26.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8383837142857142, "accuracy_std_dev": 0.026235324298762266, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11822585137085138, "avg_duration_per_task_ms": 408512.2987012987, "avg_output_tokens_per_task": 35583.771284271286, "last_run_timestamp": "2026-10-03T10:08:41.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8381875026968716, "accuracy_std_dev": 0.003092760668974499, "run_count": 6, "total_questions": 927, "avg_cost_per_task_usd": 0.13592745091693637, "avg_duration_per_task_ms": 121247.60086299892, "avg_output_tokens_per_task": 30661.07551240561, "last_run_timestamp": "2026-10-03T22:03:15.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8381033333333334, "accuracy_std_dev": 0.0831951431348142, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007828982323232323, "avg_duration_per_task_ms": 443392.71717171714, "avg_output_tokens_per_task": 28136.66245791246, "last_run_timestamp": "2026-10-03T17:03:58.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.837962875, "accuracy_std_dev": 0.08099089361378461, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007212295454545454, "avg_duration_per_task_ms": 692973.9797979798, "avg_output_tokens_per_task": 29648.13005050505, "last_run_timestamp": "2026-10-03T11:41:47.254Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8376334722439405, "accuracy_std_dev": 0.027219006084059767, "run_count": 7, "total_questions": 1279, "avg_cost_per_task_usd": 0.0889748506645817, "avg_duration_per_task_ms": 1571235.6520719312, "avg_output_tokens_per_task": 41303.13604378421, "last_run_timestamp": "2026-08-29T19:12:33.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8375589327830189, "accuracy_std_dev": 0.020056898198209345, "run_count": 9, "total_questions": 1696, "avg_cost_per_task_usd": 0.054286748820754724, "avg_duration_per_task_ms": 282159.9534198113, "avg_output_tokens_per_task": 22660.685141509435, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.837017, "accuracy_std_dev": null, "run_count": 1, "total_questions": 181, "avg_cost_per_task_usd": 0.02255581215469613, "avg_duration_per_task_ms": 455797.3591160221, "avg_output_tokens_per_task": 19569.629834254145, "last_run_timestamp": "2026-07-11T15:51:15.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8367149173913044, "accuracy_std_dev": 0.019766126930603665, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.0784595304347826, "avg_duration_per_task_ms": 736815.4384057971, "avg_output_tokens_per_task": 42336.87536231884, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8366436799091941, "accuracy_std_dev": 0.015820054069611535, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.08522557775255392, "avg_duration_per_task_ms": 546245.1770715096, "avg_output_tokens_per_task": 31173.529511918274, "last_run_timestamp": "2026-10-03T11:03:00.010Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8362222851165569, "accuracy_std_dev": 0.014310164504994333, "run_count": 10, "total_questions": 1673, "avg_cost_per_task_usd": 0.0965615690376569, "avg_duration_per_task_ms": 627840.9587567245, "avg_output_tokens_per_task": 30100.354453078304, "last_run_timestamp": "2026-10-03T17:45:46.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8360065448008849, "accuracy_std_dev": 0.008779454087962683, "run_count": 11, "total_questions": 1808, "avg_cost_per_task_usd": 0.018033527101769913, "avg_duration_per_task_ms": 187582.5735619469, "avg_output_tokens_per_task": 35877.24225663717, "last_run_timestamp": "2026-10-03T09:31:41.635Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8359790095238095, "accuracy_std_dev": 0.03660393871642171, "run_count": 6, "total_questions": 630, "avg_cost_per_task_usd": 0.10253282222222224, "avg_duration_per_task_ms": 301607.7301587302, "avg_output_tokens_per_task": 23860.25873015873, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8357464991482112, "accuracy_std_dev": 0.060956979201182, "run_count": 6, "total_questions": 1174, "avg_cost_per_task_usd": 0.08090285604770017, "avg_duration_per_task_ms": 586630.9148211244, "avg_output_tokens_per_task": 22899.433560477002, "last_run_timestamp": "2026-10-03T09:16:10.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.835648125, "accuracy_std_dev": 0.012901937649414363, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03973862878787879, "avg_duration_per_task_ms": 223092.5542929293, "avg_output_tokens_per_task": 14589.925505050505, "last_run_timestamp": "2026-10-03T09:51:44.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8356289090909089, "accuracy_std_dev": 0.029442978957653503, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.21974994260789715, "avg_duration_per_task_ms": 332523.40220385673, "avg_output_tokens_per_task": 49815.44995408632, "last_run_timestamp": "2026-10-03T17:09:33.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8354408285280729, "accuracy_std_dev": 0.043474653742611905, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.1658476181082448, "avg_duration_per_task_ms": 161596.665149216, "avg_output_tokens_per_task": 16364.909458775923, "last_run_timestamp": "2026-10-03T09:33:36.803Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8350166666666667, "accuracy_std_dev": 0.005991526757754575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06194655555555556, "avg_duration_per_task_ms": 84396.87373737374, "avg_output_tokens_per_task": 13638.5, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8348964285714285, "accuracy_std_dev": 0.013870860266422272, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13617697113997115, "avg_duration_per_task_ms": 172785.43217893218, "avg_output_tokens_per_task": 30709.33621933622, "last_run_timestamp": "2026-10-03T09:19:24.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8347762857142856, "accuracy_std_dev": 0.039321677971442086, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10741473737373737, "avg_duration_per_task_ms": 1184256.5779220778, "avg_output_tokens_per_task": 43225.256132756134, "last_run_timestamp": "2026-08-29T18:42:20.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.834656, "accuracy_std_dev": 0.01293003014028089, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006965619769119769, "avg_duration_per_task_ms": 380792.71356421354, "avg_output_tokens_per_task": 26227.490620490622, "last_run_timestamp": "2026-10-03T17:23:05.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8345736842105264, "accuracy_std_dev": 0.012051692648327507, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.2574891374269006, "avg_duration_per_task_ms": 275645.54146730463, "avg_output_tokens_per_task": 56198.80940988836, "last_run_timestamp": "2026-10-03T17:33:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.834551244519393, "accuracy_std_dev": 0.02463277117085194, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.15877912478920744, "avg_duration_per_task_ms": 826328.9359190556, "avg_output_tokens_per_task": 52149.6323777403, "last_run_timestamp": "2026-10-03T17:11:42.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.8344516577181208, "accuracy_std_dev": 0.048501499999999975, "run_count": 2, "total_questions": 298, "avg_cost_per_task_usd": 0.0903846644295302, "avg_duration_per_task_ms": 304392.4194630872, "avg_output_tokens_per_task": 22866.939597315435, "last_run_timestamp": "2026-07-18T08:27:05.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8344179826464209, "accuracy_std_dev": 0.05856560161712257, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.22784925741142445, "avg_duration_per_task_ms": 1484141.7903109184, "avg_output_tokens_per_task": 53229.01156905278, "last_run_timestamp": "2026-10-03T17:35:18.039Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "WandB", "accuracy": 0.834197, "accuracy_std_dev": null, "run_count": 1, "total_questions": 193, "avg_cost_per_task_usd": 0.02599202590673575, "avg_duration_per_task_ms": 128468.99481865285, "avg_output_tokens_per_task": 20660.492227979274, "last_run_timestamp": "2026-07-18T16:00:08.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.83417930964467, "accuracy_std_dev": 0.010994500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.033813327411167514, "avg_duration_per_task_ms": 296242.83248730964, "avg_output_tokens_per_task": 28735.548223350255, "last_run_timestamp": "2026-07-18T08:29:12.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8340991776416539, "accuracy_std_dev": 0.012440982886713574, "run_count": 4, "total_questions": 653, "avg_cost_per_task_usd": 0.04871676722817765, "avg_duration_per_task_ms": 96441.48238897396, "avg_output_tokens_per_task": 15147.393568147014, "last_run_timestamp": "2026-10-03T09:04:19.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8340809047085203, "accuracy_std_dev": 0.01732897775403963, "run_count": 5, "total_questions": 892, "avg_cost_per_task_usd": 0.020895510089686098, "avg_duration_per_task_ms": 283776.2948430493, "avg_output_tokens_per_task": 41571.40470852018, "last_run_timestamp": "2026-10-03T09:31:15.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.8339628488664986, "accuracy_std_dev": 0.062186325431239316, "run_count": 5, "total_questions": 794, "avg_cost_per_task_usd": 0.1341619395465995, "avg_duration_per_task_ms": 1520039.8690176322, "avg_output_tokens_per_task": 46924.88035264484, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.05954210180106922, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0609229090909091, "avg_duration_per_task_ms": 629153.6734006734, "avg_output_tokens_per_task": 22578.223905723906, "last_run_timestamp": "2026-10-03T16:58:19.180Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8337931059602649, "accuracy_std_dev": 0.013355036895202588, "run_count": 12, "total_questions": 1812, "avg_cost_per_task_usd": 0.03549873675496689, "avg_duration_per_task_ms": 241451.8725165563, "avg_output_tokens_per_task": 28296.954194260485, "last_run_timestamp": "2026-10-03T16:12:07.493Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8337923636363636, "accuracy_std_dev": 0.031059653654548318, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12979151698806246, "avg_duration_per_task_ms": 553080.608815427, "avg_output_tokens_per_task": 39262.16712580349, "last_run_timestamp": "2026-10-03T16:58:32.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8334570734966593, "accuracy_std_dev": 0.0419490749322732, "run_count": 7, "total_questions": 1347, "avg_cost_per_task_usd": 0.02924405196733482, "avg_duration_per_task_ms": 281996.3355605048, "avg_output_tokens_per_task": 9032.244988864142, "last_run_timestamp": "2026-10-03T10:36:42.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.8333332105263158, "accuracy_std_dev": 0.01599350000000005, "run_count": 2, "total_questions": 342, "avg_cost_per_task_usd": 0.16854262573099413, "avg_duration_per_task_ms": 685525.8274853801, "avg_output_tokens_per_task": 47357.02046783626, "last_run_timestamp": "2026-07-18T08:29:37.313Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8333331818181818, "accuracy_std_dev": 0.00830971445541776, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.08045208172635446, "avg_duration_per_task_ms": 795789.0610651975, "avg_output_tokens_per_task": 47822.912764003675, "last_run_timestamp": "2026-10-03T16:25:22.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8331929166666667, "accuracy_std_dev": 0.03339137345547584, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.055467366161616155, "avg_duration_per_task_ms": 273367.7276936027, "avg_output_tokens_per_task": 23750.69654882155, "last_run_timestamp": "2026-10-03T09:29:36.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8329019637618638, "accuracy_std_dev": 0.014650670481433776, "run_count": 6, "total_questions": 1159, "avg_cost_per_task_usd": 0.1486290198446937, "avg_duration_per_task_ms": 266639.7955133736, "avg_output_tokens_per_task": 33590.909404659185, "last_run_timestamp": "2026-10-03T10:00:12.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.8327431107738998, "accuracy_std_dev": 0.03352054924908005, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.1104804410723318, "avg_duration_per_task_ms": 986804.3333333334, "avg_output_tokens_per_task": 34519.57764289327, "last_run_timestamp": "2026-09-26T10:24:21.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.832620296969697, "accuracy_std_dev": 0.019550200858985223, "run_count": 15, "total_questions": 2805, "avg_cost_per_task_usd": 0.1869388680926916, "avg_duration_per_task_ms": 373798.6777183601, "avg_output_tokens_per_task": 38438.90623885918, "last_run_timestamp": "2026-10-03T16:59:44.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8324915000000002, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13435382575757576, "avg_duration_per_task_ms": 466692.7878787879, "avg_output_tokens_per_task": 61036.83838383838, "last_run_timestamp": "2026-10-03T17:03:50.322Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8324914999999999, "accuracy_std_dev": 0.01902788623310536, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02279124747474747, "avg_duration_per_task_ms": 532516.4444444445, "avg_output_tokens_per_task": 41107.4779040404, "last_run_timestamp": "2026-10-03T10:07:54.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.8324914999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023044303030303033, "avg_duration_per_task_ms": 640134.2070707071, "avg_output_tokens_per_task": 54011.681818181816, "last_run_timestamp": "2026-09-26T08:54:28.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8323712857142858, "accuracy_std_dev": 0.006903653726506745, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.24708544516594516, "avg_duration_per_task_ms": 346092.02308802307, "avg_output_tokens_per_task": 40400.90404040404, "last_run_timestamp": "2026-10-03T10:13:08.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8322391838074399, "accuracy_std_dev": 0.02245308913522337, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.08690679649890591, "avg_duration_per_task_ms": 951013.2662290299, "avg_output_tokens_per_task": 39963.18307804522, "last_run_timestamp": "2026-08-30T00:28:09.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8314979893324156, "accuracy_std_dev": 0.10730469177701411, "run_count": 15, "total_questions": 2906, "avg_cost_per_task_usd": 0.007892868203716447, "avg_duration_per_task_ms": 643389.9057123193, "avg_output_tokens_per_task": 27155.329318651067, "last_run_timestamp": "2026-10-03T09:52:45.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8313018608082102, "accuracy_std_dev": 0.013695449328918986, "run_count": 9, "total_questions": 1559, "avg_cost_per_task_usd": 0.11615080372033355, "avg_duration_per_task_ms": 231131.58178319436, "avg_output_tokens_per_task": 28108.629249518923, "last_run_timestamp": "2026-10-03T09:35:09.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.8312665185995624, "accuracy_std_dev": 0.02000709051964389, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.07119963603209337, "avg_duration_per_task_ms": 887802.9110138585, "avg_output_tokens_per_task": 40565.85995623632, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8310885000000001, "accuracy_std_dev": 0.012955632388399007, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06418621717171719, "avg_duration_per_task_ms": 117116.24831649831, "avg_output_tokens_per_task": 13483.090067340067, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.8309340194750211, "accuracy_std_dev": 0.0227049955198312, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.0975369652836579, "avg_duration_per_task_ms": 214278.25232853513, "avg_output_tokens_per_task": 28093.950889077052, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8308079999999999, "accuracy_std_dev": 0.019360000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.16246437626262625, "avg_duration_per_task_ms": 214673.86111111112, "avg_output_tokens_per_task": 36669.38383838384, "last_run_timestamp": "2026-10-03T09:42:09.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8307627712082263, "accuracy_std_dev": 0.005791790283452965, "run_count": 8, "total_questions": 1556, "avg_cost_per_task_usd": 0.03552349742930591, "avg_duration_per_task_ms": 197284.01863753214, "avg_output_tokens_per_task": 13956.56233933162, "last_run_timestamp": "2026-10-03T09:16:27.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8304254545454545, "accuracy_std_dev": 0.08702562758005528, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09945538659320477, "avg_duration_per_task_ms": 844364.1143250689, "avg_output_tokens_per_task": 42647.71625344353, "last_run_timestamp": "2026-10-03T16:52:01.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8302311218274112, "accuracy_std_dev": 0.009959264201340341, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.0075188776085730385, "avg_duration_per_task_ms": 370069.358714044, "avg_output_tokens_per_task": 19121.66271855612, "last_run_timestamp": "2026-10-04T00:00:36.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.829966, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022576904040404043, "avg_duration_per_task_ms": 10045.939393939394, "avg_output_tokens_per_task": 14604.111111111111, "last_run_timestamp": "2026-10-03T08:20:05.754Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8298018056195302, "accuracy_std_dev": 0.03397957835387955, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.06617050253339476, "avg_duration_per_task_ms": 1041423.7001381852, "avg_output_tokens_per_task": 33323.82865039152, "last_run_timestamp": "2026-10-03T17:39:05.807Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.82975575, "accuracy_std_dev": 0.03242017665262638, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1467550763888889, "avg_duration_per_task_ms": 981685.6369949495, "avg_output_tokens_per_task": 52890.121212121216, "last_run_timestamp": "2026-10-03T17:37:18.132Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8297257142857143, "accuracy_std_dev": 0.01460544534181185, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02442155772005772, "avg_duration_per_task_ms": 463793.1847041847, "avg_output_tokens_per_task": 22682.81313131313, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8295454999999999, "accuracy_std_dev": 0.049917598154358346, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09026368434343435, "avg_duration_per_task_ms": 461794.2853535354, "avg_output_tokens_per_task": 20466.646464646463, "last_run_timestamp": "2026-10-03T17:40:21.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.8294053333333333, "accuracy_std_dev": 0.0031744380430068155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008978000000000002, "avg_duration_per_task_ms": 380339.61616161617, "avg_output_tokens_per_task": 21414.491582491584, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8292847495662232, "accuracy_std_dev": 0.024481615398972544, "run_count": 9, "total_questions": 1729, "avg_cost_per_task_usd": 0.021951503759398496, "avg_duration_per_task_ms": 881311.056680162, "avg_output_tokens_per_task": 45066.87854251012, "last_run_timestamp": "2026-10-03T09:15:02.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8288054812419146, "accuracy_std_dev": 0.014299354744511224, "run_count": 8, "total_questions": 1546, "avg_cost_per_task_usd": 0.03668785575679172, "avg_duration_per_task_ms": 429133.35187580856, "avg_output_tokens_per_task": 46204.894566623545, "last_run_timestamp": "2026-10-03T09:52:10.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8286333058568329, "accuracy_std_dev": 0.029611903710969927, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.025387288503253794, "avg_duration_per_task_ms": 480511.26970354305, "avg_output_tokens_per_task": 23613.104121475055, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.82849325, "accuracy_std_dev": 0.05083663500800876, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09963171022727271, "avg_duration_per_task_ms": 814676.8005050505, "avg_output_tokens_per_task": 39824.98926767677, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.828283, "accuracy_std_dev": 0.016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00548029292929293, "avg_duration_per_task_ms": 1403265.9545454546, "avg_output_tokens_per_task": 25437.79292929293, "last_run_timestamp": "2026-07-18T08:42:46.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.827862, "accuracy_std_dev": 0.0497186507736282, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.16292082386363638, "avg_duration_per_task_ms": 160402.9981060606, "avg_output_tokens_per_task": 16138.333333333334, "last_run_timestamp": "2026-10-03T09:35:14.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8277963918918919, "accuracy_std_dev": 0.03021182471021881, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.01481789527027027, "avg_duration_per_task_ms": 129146.45101351352, "avg_output_tokens_per_task": 17764.291666666668, "last_run_timestamp": "2026-10-03T17:12:18.981Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.82744125, "accuracy_std_dev": 0.11256498383905848, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.10842245328282829, "avg_duration_per_task_ms": 1501711.7184343433, "avg_output_tokens_per_task": 26982.195707070707, "last_run_timestamp": "2026-10-03T16:52:29.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8274407499999998, "accuracy_std_dev": 0.002791768917281665, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06294326136363637, "avg_duration_per_task_ms": 238417.01893939395, "avg_output_tokens_per_task": 15108.617424242424, "last_run_timestamp": "2026-10-03T10:04:25.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8270997818897639, "accuracy_std_dev": 0.023868839957632546, "run_count": 13, "total_questions": 2540, "avg_cost_per_task_usd": 0.04439519330708661, "avg_duration_per_task_ms": 367160.94881889765, "avg_output_tokens_per_task": 47330.37204724409, "last_run_timestamp": "2026-10-03T17:12:02.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8267676, "accuracy_std_dev": 0.014373097433747551, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.06195891818181818, "avg_duration_per_task_ms": 501765.85909090907, "avg_output_tokens_per_task": 48813.566666666666, "last_run_timestamp": "2026-10-03T17:23:58.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.8265989999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02378419696969697, "avg_duration_per_task_ms": 421976.08585858584, "avg_output_tokens_per_task": 20387.959595959597, "last_run_timestamp": "2026-07-18T16:18:27.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8263587142857144, "accuracy_std_dev": 0.007278841366086593, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1363799206349206, "avg_duration_per_task_ms": 152940.04473304472, "avg_output_tokens_per_task": 33517.63347763348, "last_run_timestamp": "2026-10-03T09:28:23.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8262166363636364, "accuracy_std_dev": 0.05839918106566644, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.05209585123966942, "avg_duration_per_task_ms": 600270.5303030303, "avg_output_tokens_per_task": 24943.10101010101, "last_run_timestamp": "2026-10-03T08:20:09.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8261182857142857, "accuracy_std_dev": 0.010787762507506711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.043446235209235205, "avg_duration_per_task_ms": 226902.49855699856, "avg_output_tokens_per_task": 14365.386002886004, "last_run_timestamp": "2026-10-03T09:30:12.452Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8258873707224335, "accuracy_std_dev": 0.009676826643527784, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.025350776615969583, "avg_duration_per_task_ms": 120122.08840304182, "avg_output_tokens_per_task": 20151.38783269962, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.8254766666666666, "accuracy_std_dev": 0.009654676391377487, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008892052188552189, "avg_duration_per_task_ms": 1093286.5, "avg_output_tokens_per_task": 21288.892255892257, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8254038475770924, "accuracy_std_dev": 0.017871570248271126, "run_count": 6, "total_questions": 1135, "avg_cost_per_task_usd": 0.024938470484581497, "avg_duration_per_task_ms": 321173.1885462555, "avg_output_tokens_per_task": 49662.51806167401, "last_run_timestamp": "2026-10-03T09:30:56.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8253399469387755, "accuracy_std_dev": 0.06202361079266511, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.048117852040816325, "avg_duration_per_task_ms": 1077180.0714285714, "avg_output_tokens_per_task": 26155.976530612246, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8249158181818181, "accuracy_std_dev": 0.02292623538782263, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.010601494031221303, "avg_duration_per_task_ms": 636029.4292929292, "avg_output_tokens_per_task": 37631.881542699724, "last_run_timestamp": "2026-10-03T09:23:44.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8249157142857142, "accuracy_std_dev": 0.019837994431424387, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1225871738816739, "avg_duration_per_task_ms": 339042.2626262626, "avg_output_tokens_per_task": 50612.39971139971, "last_run_timestamp": "2026-09-26T08:56:23.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8246, "accuracy_std_dev": 0.010795510467319271, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09903964393939395, "avg_duration_per_task_ms": 375586.2720959596, "avg_output_tokens_per_task": 45638.01262626263, "last_run_timestamp": "2026-10-04T00:05:42.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.824444492063492, "accuracy_std_dev": 0.011481171155826198, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.02515456126984127, "avg_duration_per_task_ms": 461294.5841269841, "avg_output_tokens_per_task": 25119.664126984128, "last_run_timestamp": "2026-10-03T16:15:37.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.8243546666666668, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0070816599326599325, "avg_duration_per_task_ms": 648845.1127946128, "avg_output_tokens_per_task": 16910.563973063974, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8243546666666666, "accuracy_std_dev": 0.012396478769751095, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.038657538720538724, "avg_duration_per_task_ms": 268369.8787878788, "avg_output_tokens_per_task": 15453.594276094276, "last_run_timestamp": "2026-10-03T09:38:33.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8238636249999999, "accuracy_std_dev": 0.022648157226016746, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03663502967171717, "avg_duration_per_task_ms": 270780.9375, "avg_output_tokens_per_task": 14336.900252525253, "last_run_timestamp": "2026-10-03T09:40:25.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8237297855579868, "accuracy_std_dev": 0.03088832039090874, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.025826647702407, "avg_duration_per_task_ms": 464740.68708971556, "avg_output_tokens_per_task": 21465.053975200582, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.823569, "accuracy_std_dev": 0.003264448253533828, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.039843759595959594, "avg_duration_per_task_ms": 211450.69292929294, "avg_output_tokens_per_task": 14991.045454545454, "last_run_timestamp": "2026-10-03T09:53:48.526Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8234428750000001, "accuracy_std_dev": 0.0073110015462571785, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04899790467171717, "avg_duration_per_task_ms": 455414.46527777775, "avg_output_tokens_per_task": 27141.95202020202, "last_run_timestamp": "2026-10-03T16:27:31.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8233685148148148, "accuracy_std_dev": 0.018414157846885104, "run_count": 10, "total_questions": 1890, "avg_cost_per_task_usd": 0.0329625582010582, "avg_duration_per_task_ms": 257300.5761904762, "avg_output_tokens_per_task": 14146.621164021164, "last_run_timestamp": "2026-10-03T09:29:11.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.8233255173210161, "accuracy_std_dev": 0.0467612856991764, "run_count": 5, "total_questions": 866, "avg_cost_per_task_usd": 0.16841330484988454, "avg_duration_per_task_ms": 1763547.682448037, "avg_output_tokens_per_task": 49858.13856812933, "last_run_timestamp": "2026-10-03T17:19:48.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025912994949494946, "avg_duration_per_task_ms": 162312.898989899, "avg_output_tokens_per_task": 9036.257575757576, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8231693085966202, "accuracy_std_dev": 0.01720174744208347, "run_count": 7, "total_questions": 1361, "avg_cost_per_task_usd": 0.13977533284349744, "avg_duration_per_task_ms": 264582.22189566493, "avg_output_tokens_per_task": 31542.92799412197, "last_run_timestamp": "2026-10-03T09:52:37.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8227512857142858, "accuracy_std_dev": 0.013638696342332996, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03510758802308803, "avg_duration_per_task_ms": 250326.75108225108, "avg_output_tokens_per_task": 13920.564213564214, "last_run_timestamp": "2026-10-03T09:31:25.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8223905999999999, "accuracy_std_dev": 0.023475537008554266, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.03811036666666667, "avg_duration_per_task_ms": 320984.0297979798, "avg_output_tokens_per_task": 39454.10808080808, "last_run_timestamp": "2026-10-03T16:50:17.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.821789142857143, "accuracy_std_dev": 0.03130762753446411, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09338334415584416, "avg_duration_per_task_ms": 1431277.56998557, "avg_output_tokens_per_task": 43321.73593073593, "last_run_timestamp": "2026-10-03T17:44:28.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.8216522615384615, "accuracy_std_dev": 0.004955863351178644, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.008033046153846153, "avg_duration_per_task_ms": 616226.1504273504, "avg_output_tokens_per_task": 19179.20512820513, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8215487142857143, "accuracy_std_dev": 0.020161900680769788, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10321117243867245, "avg_duration_per_task_ms": 633991.6284271284, "avg_output_tokens_per_task": 38277.07575757576, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8212558434782609, "accuracy_std_dev": 0.006277707202019371, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.007720972463768116, "avg_duration_per_task_ms": 593072.697826087, "avg_output_tokens_per_task": 17972.839130434782, "last_run_timestamp": "2026-08-29T18:40:24.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8210436999999998, "accuracy_std_dev": 0.014005426348740687, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.3147447828282829, "avg_duration_per_task_ms": 289184.82020202023, "avg_output_tokens_per_task": 20793.45303030303, "last_run_timestamp": "2026-10-03T09:11:51.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.8209994459203036, "accuracy_std_dev": 0.013381158348004657, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.18810200189753318, "avg_duration_per_task_ms": 919106.2055660974, "avg_output_tokens_per_task": 50318.87096774193, "last_run_timestamp": "2026-10-03T16:39:56.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.820920596529284, "accuracy_std_dev": 0.007605756273362683, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.019052912509038324, "avg_duration_per_task_ms": 539380.6218365871, "avg_output_tokens_per_task": 18914.664497469268, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8207728695652173, "accuracy_std_dev": 0.01735923745738968, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.021539910869565217, "avg_duration_per_task_ms": 499775.89130434784, "avg_output_tokens_per_task": 22209.624637681158, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.82070725, "accuracy_std_dev": 0.0036691031857253717, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1439596148989899, "avg_duration_per_task_ms": 393442.0984848485, "avg_output_tokens_per_task": 28794.420454545456, "last_run_timestamp": "2026-10-03T09:54:41.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.820012649122807, "accuracy_std_dev": 0.022660477062222247, "run_count": 4, "total_questions": 513, "avg_cost_per_task_usd": 0.08905661598440547, "avg_duration_per_task_ms": 310577.5692007797, "avg_output_tokens_per_task": 36897.34307992203, "last_run_timestamp": "2026-10-03T17:40:40.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8198652, "accuracy_std_dev": 0.057672987104536215, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.027881877272727274, "avg_duration_per_task_ms": 630281.2954545454, "avg_output_tokens_per_task": 28888.00202020202, "last_run_timestamp": "2026-10-03T10:05:26.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8191919000000001, "accuracy_std_dev": 0.0203168642878275, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.012244186363636363, "avg_duration_per_task_ms": 659276.7813131313, "avg_output_tokens_per_task": 36210.856060606064, "last_run_timestamp": "2026-10-03T10:16:01.896Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.818813, "accuracy_std_dev": 0.021492541590049312, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.042027822601010095, "avg_duration_per_task_ms": 181041.06944444444, "avg_output_tokens_per_task": 23246.748106060608, "last_run_timestamp": "2026-10-03T17:30:27.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8184125485714286, "accuracy_std_dev": 0.011787855675970066, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.022701838095238094, "avg_duration_per_task_ms": 349381.28571428574, "avg_output_tokens_per_task": 20227.981587301587, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8179412857142858, "accuracy_std_dev": 0.021435433107119505, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00647507287157287, "avg_duration_per_task_ms": 501341.06277056277, "avg_output_tokens_per_task": 17357.66810966811, "last_run_timestamp": "2026-10-03T17:03:40.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8179307705544933, "accuracy_std_dev": 0.0314446048740877, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.23604747928616954, "avg_duration_per_task_ms": 962378.0981516889, "avg_output_tokens_per_task": 53848.4913957935, "last_run_timestamp": "2026-10-03T17:05:40.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8178076666666667, "accuracy_std_dev": 0.020010550661532987, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04244091638608306, "avg_duration_per_task_ms": 255376.0824915825, "avg_output_tokens_per_task": 14176.158810325476, "last_run_timestamp": "2026-10-03T10:33:26.506Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8177005714285713, "accuracy_std_dev": 0.009638124000286448, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01912177994227994, "avg_duration_per_task_ms": 549930.608946609, "avg_output_tokens_per_task": 18923.724386724385, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8165714782608695, "accuracy_std_dev": 0.012063277850998734, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.3338848906455862, "avg_duration_per_task_ms": 299201.53820816864, "avg_output_tokens_per_task": 21149.658322353975, "last_run_timestamp": "2026-10-03T09:43:11.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8160116235741445, "accuracy_std_dev": 0.012319818494376241, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.007714593789607097, "avg_duration_per_task_ms": 439013.47782002535, "avg_output_tokens_per_task": 19841.0525982256, "last_run_timestamp": "2026-10-04T00:12:29.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.815937, "accuracy_std_dev": 0.021042110493009022, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04137441245791246, "avg_duration_per_task_ms": 82874.20875420875, "avg_output_tokens_per_task": 27465.196969696968, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8158749778364116, "accuracy_std_dev": 0.06466277255569855, "run_count": 20, "total_questions": 3790, "avg_cost_per_task_usd": 0.0072713517150395784, "avg_duration_per_task_ms": 485517.56992084434, "avg_output_tokens_per_task": 17092.549340369394, "last_run_timestamp": "2026-10-03T17:48:04.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8156565, "accuracy_std_dev": 0.01038452301023016, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.36446038207070713, "avg_duration_per_task_ms": 359197.3244949495, "avg_output_tokens_per_task": 22956.0851010101, "last_run_timestamp": "2026-10-03T09:59:43.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.8152740000000002, "accuracy_std_dev": 0.03956053484563532, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.11835203581267217, "avg_duration_per_task_ms": 493386.7966023875, "avg_output_tokens_per_task": 33958.68686868687, "last_run_timestamp": "2026-10-03T16:58:37.335Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8143337142857142, "accuracy_std_dev": 0.0065951553586008314, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006175107503607504, "avg_duration_per_task_ms": 547758.3585858586, "avg_output_tokens_per_task": 18782.603174603173, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8141798076190477, "accuracy_std_dev": 0.050441669383897034, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.2300899580952381, "avg_duration_per_task_ms": 1003655.4222222222, "avg_output_tokens_per_task": 52717.56507936508, "last_run_timestamp": "2026-10-03T17:12:36.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.8140323536776214, "accuracy_std_dev": 0.032856006129862406, "run_count": 6, "total_questions": 639, "avg_cost_per_task_usd": 0.06527732394366197, "avg_duration_per_task_ms": 147029.73082942097, "avg_output_tokens_per_task": 28553.190923317685, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.814024, "accuracy_std_dev": null, "run_count": 1, "total_questions": 164, "avg_cost_per_task_usd": 0.010100225609756097, "avg_duration_per_task_ms": 515065.23780487804, "avg_output_tokens_per_task": 33245.84146341463, "last_run_timestamp": "2026-09-28T09:46:11.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8137084857142858, "accuracy_std_dev": 0.0321649704013025, "run_count": 6, "total_questions": 1155, "avg_cost_per_task_usd": 0.07626592467532467, "avg_duration_per_task_ms": 1453708.5913419914, "avg_output_tokens_per_task": 43644.7670995671, "last_run_timestamp": "2026-08-22T17:50:04.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8133964770240699, "accuracy_std_dev": 0.02361801216297875, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.006552795769511306, "avg_duration_per_task_ms": 681556.7833698031, "avg_output_tokens_per_task": 17687.919766593728, "last_run_timestamp": "2026-10-03T17:09:07.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.8133104923857867, "accuracy_std_dev": 0.012471288866833284, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.07992016920473774, "avg_duration_per_task_ms": 392391.41285956005, "avg_output_tokens_per_task": 33688.16074450085, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8131313333333333, "accuracy_std_dev": 0.02212166805545088, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2550713585858586, "avg_duration_per_task_ms": 227479.37878787878, "avg_output_tokens_per_task": 25491.23232323232, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.8130078536585367, "accuracy_std_dev": 0.0073984999999999745, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.07202867479674796, "avg_duration_per_task_ms": 332408.0243902439, "avg_output_tokens_per_task": 16877.368563685635, "last_run_timestamp": "2026-07-18T08:30:38.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8128504999999999, "accuracy_std_dev": 0.008607050515904558, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03321095707070707, "avg_duration_per_task_ms": 706759.8703703703, "avg_output_tokens_per_task": 15781.767676767677, "last_run_timestamp": "2026-10-03T09:29:30.693Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8127103749999999, "accuracy_std_dev": 0.009056161989737969, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03019831565656566, "avg_duration_per_task_ms": 209380.85795454544, "avg_output_tokens_per_task": 23957.397095959597, "last_run_timestamp": "2026-10-03T17:45:01.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8122894999999999, "accuracy_std_dev": 0.020078789204448232, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006274926767676768, "avg_duration_per_task_ms": 514195.62205387204, "avg_output_tokens_per_task": 18652.397306397306, "last_run_timestamp": "2026-10-03T16:58:24.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8112072857142857, "accuracy_std_dev": 0.030080678790376323, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014970598124098124, "avg_duration_per_task_ms": 908622.7445887446, "avg_output_tokens_per_task": 44238.631313131315, "last_run_timestamp": "2026-10-03T08:04:44.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8111109391304346, "accuracy_std_dev": 0.03310361291213897, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.07639000724637682, "avg_duration_per_task_ms": 954401.9471014492, "avg_output_tokens_per_task": 39369.77246376812, "last_run_timestamp": "2026-08-30T00:30:19.495Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.8105691219512196, "accuracy_std_dev": 0.043372999999999995, "run_count": 2, "total_questions": 205, "avg_cost_per_task_usd": 0.04714223902439025, "avg_duration_per_task_ms": 880440.2341463414, "avg_output_tokens_per_task": 20870.317073170732, "last_run_timestamp": "2026-07-18T16:43:10.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8103253333333333, "accuracy_std_dev": 0.011527714209195522, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04107354882154882, "avg_duration_per_task_ms": 75169.15151515152, "avg_output_tokens_per_task": 27258.996632996634, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.8100831277533042, "accuracy_std_dev": 0.021002816511805526, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.16732624743024965, "avg_duration_per_task_ms": 899402.9625550661, "avg_output_tokens_per_task": 53272.42290748899, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8099512222222223, "accuracy_std_dev": 0.02558631077040038, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10966612121212123, "avg_duration_per_task_ms": 777813.3344556678, "avg_output_tokens_per_task": 56577.68855218855, "last_run_timestamp": "2026-10-03T17:13:30.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8095018965517242, "accuracy_std_dev": 0.0486761308529131, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.08429347448275862, "avg_duration_per_task_ms": 792182.6556321839, "avg_output_tokens_per_task": 31699.048275862067, "last_run_timestamp": "2026-10-03T19:00:29.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8092201778202676, "accuracy_std_dev": 0.025362105349011843, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.19599221542383682, "avg_duration_per_task_ms": 884302.3785850861, "avg_output_tokens_per_task": 53656.268961121736, "last_run_timestamp": "2026-10-03T17:03:42.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8091166153846153, "accuracy_std_dev": 0.03183613126622014, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.19830280854700855, "avg_duration_per_task_ms": 1060541.5008547008, "avg_output_tokens_per_task": 55592.3264957265, "last_run_timestamp": "2026-10-03T16:58:49.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8087552731239093, "accuracy_std_dev": 0.0546199990705989, "run_count": 6, "total_questions": 1146, "avg_cost_per_task_usd": 0.024335609075043632, "avg_duration_per_task_ms": 1150538.0907504363, "avg_output_tokens_per_task": 49299.98254799302, "last_run_timestamp": "2026-10-03T10:11:21.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.808728230769231, "accuracy_std_dev": 0.016181671721919724, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.34003047086247085, "avg_duration_per_task_ms": 354646.60295260296, "avg_output_tokens_per_task": 22500.726495726496, "last_run_timestamp": "2026-10-03T09:08:27.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.808081, "accuracy_std_dev": 0.018027342695657245, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.044340436026936034, "avg_duration_per_task_ms": 217888.1734006734, "avg_output_tokens_per_task": 22075.030303030304, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.8079455869980879, "accuracy_std_dev": 0.03309943783864578, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.2356793072020395, "avg_duration_per_task_ms": 903521.8578712556, "avg_output_tokens_per_task": 59416.81580624601, "last_run_timestamp": "2026-10-03T16:51:04.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8073004516129032, "accuracy_std_dev": 0.05300431140963913, "run_count": 10, "total_questions": 1767, "avg_cost_per_task_usd": 0.046523333333333326, "avg_duration_per_task_ms": 546722.9988681381, "avg_output_tokens_per_task": 25674.418222976798, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.8072389999999999, "accuracy_std_dev": 0.0870155432092451, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.010984919191919193, "avg_duration_per_task_ms": 659527.0782828283, "avg_output_tokens_per_task": 25386.00420875421, "last_run_timestamp": "2026-10-03T17:34:53.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.807121686731666, "accuracy_std_dev": 0.1063175543594388, "run_count": 12, "total_questions": 2359, "avg_cost_per_task_usd": 0.047212891055532, "avg_duration_per_task_ms": 190020.95082662144, "avg_output_tokens_per_task": 22597.004662992793, "last_run_timestamp": "2026-10-03T10:14:18.163Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "WandB", "accuracy": 0.8068935127478752, "accuracy_std_dev": 0.009178499999999978, "run_count": 2, "total_questions": 353, "avg_cost_per_task_usd": 0.033640501416430595, "avg_duration_per_task_ms": 174824.1133144476, "avg_output_tokens_per_task": 26792.110481586402, "last_run_timestamp": "2026-07-18T08:04:57.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8065657, "accuracy_std_dev": 0.022453683925137975, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.3406123287878787, "avg_duration_per_task_ms": 355782.31666666665, "avg_output_tokens_per_task": 22552.313636363637, "last_run_timestamp": "2026-10-03T09:49:05.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.806210222222222, "accuracy_std_dev": 0.03697273211969953, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.25474633557800225, "avg_duration_per_task_ms": 231596.29349046017, "avg_output_tokens_per_task": 25466.63243546577, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.805555546875, "accuracy_std_dev": 0.01288499999999998, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.0362872109375, "avg_duration_per_task_ms": 729621.7864583334, "avg_output_tokens_per_task": 36271.864583333336, "last_run_timestamp": "2026-09-12T16:45:54.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.8053298279132792, "accuracy_std_dev": 0.03337153373515719, "run_count": 7, "total_questions": 738, "avg_cost_per_task_usd": 0.029580651761517616, "avg_duration_per_task_ms": 101041.89430894308, "avg_output_tokens_per_task": 30707.855013550135, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.8045310368763557, "accuracy_std_dev": 0.02305431337923024, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.027800933477946495, "avg_duration_per_task_ms": 458826.73535791755, "avg_output_tokens_per_task": 23174.896601590746, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8042325714285714, "accuracy_std_dev": 0.02234715177030169, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007311367243867243, "avg_duration_per_task_ms": 505999.5836940837, "avg_output_tokens_per_task": 18157.529581529583, "last_run_timestamp": "2026-08-29T19:12:36.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.8034189230769232, "accuracy_std_dev": 0.024712500000000026, "run_count": 2, "total_questions": 195, "avg_cost_per_task_usd": 0.08983526153846154, "avg_duration_per_task_ms": 329694.73846153845, "avg_output_tokens_per_task": 37199.13846153846, "last_run_timestamp": "2026-10-03T16:53:50.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8032406250000002, "accuracy_std_dev": 0.023560394356512266, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3350150732323232, "avg_duration_per_task_ms": 352876.84217171714, "avg_output_tokens_per_task": 22221.229166666668, "last_run_timestamp": "2026-10-03T09:34:21.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8026253078034682, "accuracy_std_dev": 0.11466736789371888, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.12527993786127167, "avg_duration_per_task_ms": 694994.5281791907, "avg_output_tokens_per_task": 39948.101156069366, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8020951111111111, "accuracy_std_dev": 0.027492606765554845, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04329651571268237, "avg_duration_per_task_ms": 84651.98765432098, "avg_output_tokens_per_task": 26957.542087542086, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.8019926891304349, "accuracy_std_dev": 0.02443714833108715, "run_count": 6, "total_questions": 920, "avg_cost_per_task_usd": 0.05089078369565218, "avg_duration_per_task_ms": 796111.2945652173, "avg_output_tokens_per_task": 22057.085869565217, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8016385562742562, "accuracy_std_dev": 0.007745278218856984, "run_count": 4, "total_questions": 773, "avg_cost_per_task_usd": 0.0594386532988357, "avg_duration_per_task_ms": 220979.64165588617, "avg_output_tokens_per_task": 17329.77360931436, "last_run_timestamp": "2026-09-26T08:53:08.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8013468181818181, "accuracy_std_dev": 0.10160515424186116, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09367279752066117, "avg_duration_per_task_ms": 528447.3296602387, "avg_output_tokens_per_task": 34556.12350780533, "last_run_timestamp": "2026-10-03T17:43:17.222Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.8007853333333332, "accuracy_std_dev": 0.008398773614178595, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0439676531986532, "avg_duration_per_task_ms": 201662.1632996633, "avg_output_tokens_per_task": 21888.12962962963, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8007155, "accuracy_std_dev": 0.04390787837905629, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.039541025883838385, "avg_duration_per_task_ms": 942411.1104797979, "avg_output_tokens_per_task": 24048.43876262626, "last_run_timestamp": "2026-09-26T09:34:13.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.800505, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.14201096464646465, "avg_duration_per_task_ms": 1387672.616161616, "avg_output_tokens_per_task": 46225.35353535353, "last_run_timestamp": "2026-07-11T16:02:27.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7996632000000001, "accuracy_std_dev": 0.17472434887834037, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09227538585858586, "avg_duration_per_task_ms": 490452.89090909093, "avg_output_tokens_per_task": 24835.06313131313, "last_run_timestamp": "2026-10-03T16:43:18.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7996631666666666, "accuracy_std_dev": 0.024569953550469384, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.045980551346801346, "avg_duration_per_task_ms": 194150.27525252526, "avg_output_tokens_per_task": 21808.128787878788, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.799242375, "accuracy_std_dev": 0.022425038149006003, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02821298611111111, "avg_duration_per_task_ms": 287144.1205808081, "avg_output_tokens_per_task": 18650.696338383837, "last_run_timestamp": "2026-10-04T00:12:29.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7989418571428571, "accuracy_std_dev": 0.010175039872558895, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01636965007215007, "avg_duration_per_task_ms": 355837.20057720057, "avg_output_tokens_per_task": 15678.383116883117, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7984005000000001, "accuracy_std_dev": 0.04195903080386868, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02730416792929293, "avg_duration_per_task_ms": 362984.9494949495, "avg_output_tokens_per_task": 25613.62626262626, "last_run_timestamp": "2026-10-03T19:56:06.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.7979796666666666, "accuracy_std_dev": 0.009816464683830397, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04952355303030303, "avg_duration_per_task_ms": 458017.70707070705, "avg_output_tokens_per_task": 24647.512626262625, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7974987142857143, "accuracy_std_dev": 0.009164318316387839, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01350666378066378, "avg_duration_per_task_ms": 299956.4531024531, "avg_output_tokens_per_task": 14063.395382395382, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Fireworks", "accuracy": 0.797043, "accuracy_std_dev": null, "run_count": 1, "total_questions": 124, "avg_cost_per_task_usd": 0.11967770967741935, "avg_duration_per_task_ms": 375576.0080645161, "avg_output_tokens_per_task": 29097.58064516129, "last_run_timestamp": "2026-07-18T08:32:17.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.79671725, "accuracy_std_dev": 0.006560703749408305, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05368239015151515, "avg_duration_per_task_ms": 1192594.2481060605, "avg_output_tokens_per_task": 24468.87436868687, "last_run_timestamp": "2026-10-03T18:46:58.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7964375923664123, "accuracy_std_dev": 0.11378331267012533, "run_count": 4, "total_questions": 655, "avg_cost_per_task_usd": 0.011028116030534352, "avg_duration_per_task_ms": 270893.28854961833, "avg_output_tokens_per_task": 23218.567938931297, "last_run_timestamp": "2026-09-12T08:53:48.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7963679903288201, "accuracy_std_dev": 0.04084804193727499, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.1404589497098646, "avg_duration_per_task_ms": 1543378.2327530626, "avg_output_tokens_per_task": 46550.5467440361, "last_run_timestamp": "2026-10-03T16:58:27.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.7956650000000001, "accuracy_std_dev": 0.0069381316108877815, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.014228089646464647, "avg_duration_per_task_ms": 327580.9936868687, "avg_output_tokens_per_task": 14697.917297979799, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7956228, "accuracy_std_dev": 0.021522407276138952, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006844225252525253, "avg_duration_per_task_ms": 363476.91515151516, "avg_output_tokens_per_task": 16652.98888888889, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.795444345060893, "accuracy_std_dev": 0.01127012798285806, "run_count": 8, "total_questions": 1478, "avg_cost_per_task_usd": 0.04254500879566983, "avg_duration_per_task_ms": 730941.7388362652, "avg_output_tokens_per_task": 19660.586603518266, "last_run_timestamp": "2026-10-03T16:16:31.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7949320752351098, "accuracy_std_dev": 0.09853873704264732, "run_count": 10, "total_questions": 1914, "avg_cost_per_task_usd": 0.08235360710553814, "avg_duration_per_task_ms": 843777.6013584117, "avg_output_tokens_per_task": 30560.44775339603, "last_run_timestamp": "2026-10-03T19:06:55.619Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7947657272727272, "accuracy_std_dev": 0.03887966022132421, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.042441610651974285, "avg_duration_per_task_ms": 470794.4618916437, "avg_output_tokens_per_task": 42513.95821854913, "last_run_timestamp": "2026-10-03T17:31:01.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7947205871946083, "accuracy_std_dev": 0.04278072577075129, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.1992195315922494, "avg_duration_per_task_ms": 633124.2401010952, "avg_output_tokens_per_task": 47420.48188711036, "last_run_timestamp": "2026-10-03T17:09:08.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7929293333333333, "accuracy_std_dev": 0.008361060791284552, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01913193771043771, "avg_duration_per_task_ms": 454912.569023569, "avg_output_tokens_per_task": 25421.24579124579, "last_run_timestamp": "2026-10-03T18:54:19.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7928527272727273, "accuracy_std_dev": 0.04874799814088574, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.17375368870523417, "avg_duration_per_task_ms": 837428.4205693296, "avg_output_tokens_per_task": 56964.92470156107, "last_run_timestamp": "2026-10-03T17:12:34.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.7924422284263961, "accuracy_std_dev": 0.011171723004482763, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.016949575296108292, "avg_duration_per_task_ms": 173392.4331641286, "avg_output_tokens_per_task": 6588.478849407784, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7918084721906924, "accuracy_std_dev": 0.14984325869964388, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.016061496027241773, "avg_duration_per_task_ms": 528116.8059023836, "avg_output_tokens_per_task": 25582.743473325765, "last_run_timestamp": "2026-10-03T08:06:43.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7915958188529246, "accuracy_std_dev": 0.016282913854712845, "run_count": 10, "total_questions": 1761, "avg_cost_per_task_usd": 0.004947599091425326, "avg_duration_per_task_ms": 1178125.1942078364, "avg_output_tokens_per_task": 32554.07495741056, "last_run_timestamp": "2026-09-26T21:21:20.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.7914820056925996, "accuracy_std_dev": 0.03882771144407526, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.014992018342821002, "avg_duration_per_task_ms": 477486.2156862745, "avg_output_tokens_per_task": 15890.679316888045, "last_run_timestamp": "2026-10-04T00:00:36.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7912458333333334, "accuracy_std_dev": 0.01576267340287031, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01776989141414141, "avg_duration_per_task_ms": 61590.219696969696, "avg_output_tokens_per_task": 5770.594276094276, "last_run_timestamp": "2026-10-03T16:56:50.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7910545355932204, "accuracy_std_dev": 0.011679465336358934, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.018264465536723163, "avg_duration_per_task_ms": 145761.14463276835, "avg_output_tokens_per_task": 15068.749717514123, "last_run_timestamp": "2026-10-03T09:28:19.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7910339219088935, "accuracy_std_dev": 0.007596404447540465, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.18853212509038325, "avg_duration_per_task_ms": 939271.1323210412, "avg_output_tokens_per_task": 47171.86912509039, "last_run_timestamp": "2026-10-03T17:06:15.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7891350392405064, "accuracy_std_dev": 0.020945155322890284, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.06166090379746835, "avg_duration_per_task_ms": 176165.68037974683, "avg_output_tokens_per_task": 19378.685443037975, "last_run_timestamp": "2026-10-03T10:40:10.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7884000133843213, "accuracy_std_dev": 0.06448360442960967, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.21083066475462078, "avg_duration_per_task_ms": 943051.9751434034, "avg_output_tokens_per_task": 64724.986615678776, "last_run_timestamp": "2026-10-03T17:10:01.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.78761423645894, "accuracy_std_dev": 0.05455970909816173, "run_count": 11, "total_questions": 1717, "avg_cost_per_task_usd": 0.05547815375655212, "avg_duration_per_task_ms": 804006.846825859, "avg_output_tokens_per_task": 46685.46825859057, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.787571717325228, "accuracy_std_dev": 0.005353629893819705, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.01375579837892604, "avg_duration_per_task_ms": 469828.6362715299, "avg_output_tokens_per_task": 14334.128672745694, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7857744999999999, "accuracy_std_dev": 0.05357356939993824, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.18102450063131315, "avg_duration_per_task_ms": 1433174.245580808, "avg_output_tokens_per_task": 45945.68939393939, "last_run_timestamp": "2026-10-03T17:00:45.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7847189879865448, "accuracy_std_dev": 0.015912532710814054, "run_count": 11, "total_questions": 2081, "avg_cost_per_task_usd": 0.008999504565112927, "avg_duration_per_task_ms": 423343.8265257088, "avg_output_tokens_per_task": 35697.484863046615, "last_run_timestamp": "2026-10-03T18:48:40.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.784512, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02099649242424242, "avg_duration_per_task_ms": 284075.095959596, "avg_output_tokens_per_task": 12479.631313131313, "last_run_timestamp": "2026-07-18T16:16:42.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.784512, "accuracy_std_dev": 0.043771000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.030820969696969697, "avg_duration_per_task_ms": 734052.446969697, "avg_output_tokens_per_task": 25681.72474747475, "last_run_timestamp": "2026-07-18T09:15:42.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7842310833333334, "accuracy_std_dev": 0.022077051155964113, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.004981357744107744, "avg_duration_per_task_ms": 268156.8005050505, "avg_output_tokens_per_task": 32664.053030303032, "last_run_timestamp": "2026-10-03T17:02:47.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.7830773446153846, "accuracy_std_dev": 0.015297999999999978, "run_count": 2, "total_questions": 325, "avg_cost_per_task_usd": 0.07515540923076923, "avg_duration_per_task_ms": 234345.21538461538, "avg_output_tokens_per_task": 20795.20923076923, "last_run_timestamp": "2026-07-18T08:37:00.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7825476666666666, "accuracy_std_dev": 0.020923567929224913, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.055650603535353546, "avg_duration_per_task_ms": 666314.5917508417, "avg_output_tokens_per_task": 24452.340909090908, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.782495077073808, "accuracy_std_dev": 0.1325801523111939, "run_count": 8, "total_questions": 1531, "avg_cost_per_task_usd": 0.023154647289353365, "avg_duration_per_task_ms": 345079.2540822991, "avg_output_tokens_per_task": 23624.08883082952, "last_run_timestamp": "2026-10-03T19:19:20.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7822107419689119, "accuracy_std_dev": 0.054227012196505914, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.007577522279792746, "avg_duration_per_task_ms": 591909.3450777202, "avg_output_tokens_per_task": 17266.958549222796, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7817837115749525, "accuracy_std_dev": 0.04901063227759871, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.13180700759013284, "avg_duration_per_task_ms": 1013578.3162555344, "avg_output_tokens_per_task": 60177.89626818469, "last_run_timestamp": "2026-10-03T17:38:07.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7815378781725889, "accuracy_std_dev": 0.023695801760263955, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.05647687704455725, "avg_duration_per_task_ms": 566671.4066553863, "avg_output_tokens_per_task": 24552.29046813311, "last_run_timestamp": "2026-10-03T16:18:18.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7804232857142855, "accuracy_std_dev": 0.00964416607835736, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.021076865079365077, "avg_duration_per_task_ms": 281668.99494949495, "avg_output_tokens_per_task": 12536.775613275613, "last_run_timestamp": "2026-10-04T00:50:16.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7799144730769231, "accuracy_std_dev": 0.07491524224415215, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.2146196794871795, "avg_duration_per_task_ms": 1130627.516025641, "avg_output_tokens_per_task": 68716.01923076923, "last_run_timestamp": "2026-10-03T16:59:22.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7794466007905138, "accuracy_std_dev": 0.015712587779687186, "run_count": 8, "total_questions": 1265, "avg_cost_per_task_usd": 0.05468981976284586, "avg_duration_per_task_ms": 1028648.5209486166, "avg_output_tokens_per_task": 28503.08932806324, "last_run_timestamp": "2026-10-03T18:12:08.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7791806666666666, "accuracy_std_dev": 0.11069114794126743, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.14781284764309766, "avg_duration_per_task_ms": 1097204.1927609427, "avg_output_tokens_per_task": 42915.851851851854, "last_run_timestamp": "2026-10-03T17:43:01.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.777456579479769, "accuracy_std_dev": 0.02062598971775015, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.021605648843930635, "avg_duration_per_task_ms": 786693.5744219653, "avg_output_tokens_per_task": 37390.66040462428, "last_run_timestamp": "2026-10-03T16:57:26.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7772166666666667, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013080006734006736, "avg_duration_per_task_ms": 67697.39225589225, "avg_output_tokens_per_task": 10341.513468013469, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.77567325, "accuracy_std_dev": 0.010934638822910419, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013401992424242422, "avg_duration_per_task_ms": 67621.48106060606, "avg_output_tokens_per_task": 10598.125, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7756132857142857, "accuracy_std_dev": 0.020236370423531182, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015738134199134196, "avg_duration_per_task_ms": 439103.9264069264, "avg_output_tokens_per_task": 15898.891053391053, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.774411, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007553676767676767, "avg_duration_per_task_ms": 145590.66666666666, "avg_output_tokens_per_task": 30019.929292929293, "last_run_timestamp": "2026-10-03T16:43:00.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7744108749999999, "accuracy_std_dev": 0.03582138896119713, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05256629166666667, "avg_duration_per_task_ms": 881480.1900252525, "avg_output_tokens_per_task": 23763.922348484848, "last_run_timestamp": "2026-10-03T18:36:41.004Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7741623318872016, "accuracy_std_dev": 0.020647498917569188, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.05963888141720896, "avg_duration_per_task_ms": 613056.7744034707, "avg_output_tokens_per_task": 27371.603036876357, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7738971985294117, "accuracy_std_dev": 0.026639499999999983, "run_count": 2, "total_questions": 272, "avg_cost_per_task_usd": 0.009933933823529412, "avg_duration_per_task_ms": 1695099, "avg_output_tokens_per_task": 35217.91176470588, "last_run_timestamp": "2026-10-03T12:01:54.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7725740909090909, "accuracy_std_dev": 0.024590222227006273, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0059262144168962345, "avg_duration_per_task_ms": 733314.9329660239, "avg_output_tokens_per_task": 37199.05142332415, "last_run_timestamp": "2026-10-03T19:04:58.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7722011249999999, "accuracy_std_dev": 0.05713206000013804, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10372384911616161, "avg_duration_per_task_ms": 556419.0063131313, "avg_output_tokens_per_task": 31124.368686868685, "last_run_timestamp": "2026-10-03T16:59:42.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7713930343283582, "accuracy_std_dev": 0.03931269277500872, "run_count": 6, "total_questions": 670, "avg_cost_per_task_usd": 0.012700616417910445, "avg_duration_per_task_ms": 53684.10149253732, "avg_output_tokens_per_task": 16854.058208955223, "last_run_timestamp": "2026-10-03T17:06:37.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7711041709956711, "accuracy_std_dev": 0.019798063555812727, "run_count": 5, "total_questions": 924, "avg_cost_per_task_usd": 0.020921933982683983, "avg_duration_per_task_ms": 243233.94264069264, "avg_output_tokens_per_task": 27811.309523809523, "last_run_timestamp": "2026-10-03T16:58:21.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7707632222222222, "accuracy_std_dev": 0.010632614017235168, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.014933182940516276, "avg_duration_per_task_ms": 152284.73961840628, "avg_output_tokens_per_task": 29838.40684624018, "last_run_timestamp": "2026-10-03T17:01:22.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7706395810055866, "accuracy_std_dev": 0.012812584360697848, "run_count": 3, "total_questions": 537, "avg_cost_per_task_usd": 0.014063508379888267, "avg_duration_per_task_ms": 616967.677839851, "avg_output_tokens_per_task": 42632.232774674114, "last_run_timestamp": "2026-10-03T17:47:01.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.770202, "accuracy_std_dev": 0.008625232557251217, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013833271043771043, "avg_duration_per_task_ms": 119359.86616161616, "avg_output_tokens_per_task": 10422.487373737375, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7696007142857143, "accuracy_std_dev": 0.01982640472929463, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.018328969696969694, "avg_duration_per_task_ms": 935948.049062049, "avg_output_tokens_per_task": 45779.92929292929, "last_run_timestamp": "2026-10-03T16:16:35.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7688791428571428, "accuracy_std_dev": 0.026826310461552323, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.017481934343434345, "avg_duration_per_task_ms": 620493.1940836941, "avg_output_tokens_per_task": 43809.34992784993, "last_run_timestamp": "2026-10-03T16:23:37.615Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.7688491071428571, "accuracy_std_dev": 0.036143499999999995, "run_count": 2, "total_questions": 168, "avg_cost_per_task_usd": 0.013457690476190477, "avg_duration_per_task_ms": 239500.45238095237, "avg_output_tokens_per_task": 24312.886904761905, "last_run_timestamp": "2026-07-18T09:01:08.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7688279969604862, "accuracy_std_dev": 0.009398481531077219, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.019866108409321178, "avg_duration_per_task_ms": 104728.69604863222, "avg_output_tokens_per_task": 5941.306484295846, "last_run_timestamp": "2026-10-03T13:44:15.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.7685897384615384, "accuracy_std_dev": 0.017564000000000024, "run_count": 2, "total_questions": 260, "avg_cost_per_task_usd": 0.008261646153846154, "avg_duration_per_task_ms": 560681.2384615385, "avg_output_tokens_per_task": 27419.23076923077, "last_run_timestamp": "2026-10-03T16:10:46.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.768458045751634, "accuracy_std_dev": 0.007179978625401643, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.01120065286855483, "avg_duration_per_task_ms": 926810.7356572258, "avg_output_tokens_per_task": 11933.642701525054, "last_run_timestamp": "2026-08-30T01:20:44.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7676545740944017, "accuracy_std_dev": 0.04018594976550331, "run_count": 6, "total_questions": 911, "avg_cost_per_task_usd": 0.14126989352360045, "avg_duration_per_task_ms": 1023387.3852908892, "avg_output_tokens_per_task": 40525.553238199784, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.76748545750332, "accuracy_std_dev": 0.010027388436645664, "run_count": 8, "total_questions": 1506, "avg_cost_per_task_usd": 0.02429856507304117, "avg_duration_per_task_ms": 72708.38977423639, "avg_output_tokens_per_task": 25460.314741035858, "last_run_timestamp": "2026-10-03T17:11:16.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.765993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08016306060606061, "avg_duration_per_task_ms": 425226.22222222225, "avg_output_tokens_per_task": 69822.58585858585, "last_run_timestamp": "2026-07-18T08:29:23.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7659136443148689, "accuracy_std_dev": 0.08670660767325684, "run_count": 7, "total_questions": 1372, "avg_cost_per_task_usd": 0.04918383236151604, "avg_duration_per_task_ms": 1637065.3381924198, "avg_output_tokens_per_task": 18412.217201166182, "last_run_timestamp": "2026-10-03T18:57:53.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7657003043478261, "accuracy_std_dev": 0.07140090192078836, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.22668331956521737, "avg_duration_per_task_ms": 1257437.0898550726, "avg_output_tokens_per_task": 72600.84420289854, "last_run_timestamp": "2026-10-03T16:45:32.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7652910342857142, "accuracy_std_dev": 0.032967884417073254, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.11766670793650795, "avg_duration_per_task_ms": 799679.3631746032, "avg_output_tokens_per_task": 45940.76698412698, "last_run_timestamp": "2026-10-03T16:58:32.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7652582599231755, "accuracy_std_dev": 0.017480436789719386, "run_count": 8, "total_questions": 1562, "avg_cost_per_task_usd": 0.008710414212548015, "avg_duration_per_task_ms": 425255.4129321383, "avg_output_tokens_per_task": 26929.24647887324, "last_run_timestamp": "2026-10-03T17:35:16.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7647939808988764, "accuracy_std_dev": 0.09845223455671283, "run_count": 10, "total_questions": 890, "avg_cost_per_task_usd": 0.003642867415730337, "avg_duration_per_task_ms": 208810.34157303371, "avg_output_tokens_per_task": 15849.9202247191, "last_run_timestamp": "2026-10-03T16:12:33.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7647061294117647, "accuracy_std_dev": 0.11983655888350603, "run_count": 5, "total_questions": 680, "avg_cost_per_task_usd": 0.015231964705882355, "avg_duration_per_task_ms": 558302.2632352941, "avg_output_tokens_per_task": 21406.642647058823, "last_run_timestamp": "2026-09-26T10:43:59.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7642006242726517, "accuracy_std_dev": 0.029173475833436126, "run_count": 7, "total_questions": 1203, "avg_cost_per_task_usd": 0.04647462261014131, "avg_duration_per_task_ms": 263437.0906068163, "avg_output_tokens_per_task": 17506.297589359932, "last_run_timestamp": "2026-10-03T09:46:23.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7626263333333334, "accuracy_std_dev": 0.021573990312410928, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.01664038832772166, "avg_duration_per_task_ms": 1086454.638047138, "avg_output_tokens_per_task": 48174.952861952865, "last_run_timestamp": "2026-10-03T17:13:23.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7622871058394161, "accuracy_std_dev": 0.08486398602541866, "run_count": 7, "total_questions": 1370, "avg_cost_per_task_usd": 0.13712116569343066, "avg_duration_per_task_ms": 1005369.0547445256, "avg_output_tokens_per_task": 37305.686131386865, "last_run_timestamp": "2026-10-03T16:47:49.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.760943, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08179280303030302, "avg_duration_per_task_ms": 426340.89898989897, "avg_output_tokens_per_task": 70994.31818181818, "last_run_timestamp": "2026-07-18T08:26:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.760732375, "accuracy_std_dev": 0.007014241975037307, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017655010101010102, "avg_duration_per_task_ms": 83495.94002525252, "avg_output_tokens_per_task": 29184.116792929293, "last_run_timestamp": "2026-10-03T16:55:04.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7607321250000001, "accuracy_std_dev": 0.04893340233275604, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.011686217171717172, "avg_duration_per_task_ms": 765424.8869949495, "avg_output_tokens_per_task": 35637.83775252525, "last_run_timestamp": "2026-10-03T10:19:08.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7605754720812183, "accuracy_std_dev": 0.010447569992586773, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0025078527918781725, "avg_duration_per_task_ms": 100160.06345177664, "avg_output_tokens_per_task": 31388.34771573604, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.7605443265306122, "accuracy_std_dev": 0.0343317834601635, "run_count": 3, "total_questions": 490, "avg_cost_per_task_usd": 0.0037374408163265305, "avg_duration_per_task_ms": 668153.0571428571, "avg_output_tokens_per_task": 14206.583673469388, "last_run_timestamp": "2026-08-08T11:42:26.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7592001072112318, "accuracy_std_dev": 0.11665676901706133, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.17622283024888324, "avg_duration_per_task_ms": 1196115.2144224632, "avg_output_tokens_per_task": 51216.97766432674, "last_run_timestamp": "2026-10-03T18:23:36.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7590187142857142, "accuracy_std_dev": 0.03157783272538906, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.168772468975469, "avg_duration_per_task_ms": 557901.2611832612, "avg_output_tokens_per_task": 52553.58585858586, "last_run_timestamp": "2026-10-03T17:03:01.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7590111668383109, "accuracy_std_dev": 0.016611473377157127, "run_count": 5, "total_questions": 971, "avg_cost_per_task_usd": 0.004071515962924819, "avg_duration_per_task_ms": 4054.4541709577757, "avg_output_tokens_per_task": 2252.576725025747, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.758804538083538, "accuracy_std_dev": 0.09270215339222468, "run_count": 3, "total_questions": 407, "avg_cost_per_task_usd": 0.03842371253071253, "avg_duration_per_task_ms": 1486381.9508599509, "avg_output_tokens_per_task": 26977.572481572483, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.758698, "accuracy_std_dev": 0.012009662443216293, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.056970961279461274, "avg_duration_per_task_ms": 99388.35016835017, "avg_output_tokens_per_task": 22699.07239057239, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7575759999999999, "accuracy_std_dev": 0.017346370936500422, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005004900673400673, "avg_duration_per_task_ms": 851788.595959596, "avg_output_tokens_per_task": 28068.07744107744, "last_run_timestamp": "2026-10-03T17:30:01.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.75757575, "accuracy_std_dev": 0.010030692819416823, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01872427398989899, "avg_duration_per_task_ms": 177720.84722222222, "avg_output_tokens_per_task": 31014.384469696968, "last_run_timestamp": "2026-10-03T16:34:22.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7569444375000001, "accuracy_std_dev": 0.012152708031385188, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.018743317234848483, "avg_duration_per_task_ms": 181792.14551767678, "avg_output_tokens_per_task": 31017.58143939394, "last_run_timestamp": "2026-10-03T16:20:04.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7568806935779815, "accuracy_std_dev": 0.10522151155327296, "run_count": 3, "total_questions": 545, "avg_cost_per_task_usd": 0.004833203669724771, "avg_duration_per_task_ms": 268953.0513761468, "avg_output_tokens_per_task": 24299.52110091743, "last_run_timestamp": "2026-10-01T16:23:05.513Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.7565450000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.020916916230366494, "avg_duration_per_task_ms": 124793.11518324607, "avg_output_tokens_per_task": 18115.47120418848, "last_run_timestamp": "2026-07-18T08:26:49.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7550891068702291, "accuracy_std_dev": 0.013103736509103058, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.01650294465648855, "avg_duration_per_task_ms": 296983.9370229008, "avg_output_tokens_per_task": 28161.054707379135, "last_run_timestamp": "2026-10-03T17:05:00.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7548401249999999, "accuracy_std_dev": 0.04325769864844148, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05385063825757575, "avg_duration_per_task_ms": 956346.2373737374, "avg_output_tokens_per_task": 27778.031565656565, "last_run_timestamp": "2026-10-03T17:12:39.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7545347341659232, "accuracy_std_dev": 0.02761225844139473, "run_count": 8, "total_questions": 1121, "avg_cost_per_task_usd": 0.05582876717216771, "avg_duration_per_task_ms": 404528.79750223015, "avg_output_tokens_per_task": 16141.22033898305, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7542087500000001, "accuracy_std_dev": 0.008073547791243961, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022397893939393936, "avg_duration_per_task_ms": 97901.73674242424, "avg_output_tokens_per_task": 14784.859217171717, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7539683918918919, "accuracy_std_dev": 0.07441117537362499, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.131239731016731, "avg_duration_per_task_ms": 1027517.648005148, "avg_output_tokens_per_task": 56777.77670527671, "last_run_timestamp": "2026-10-03T16:56:39.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.753934655143339, "accuracy_std_dev": 0.024433692146069302, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.020737796795952786, "avg_duration_per_task_ms": 145659.42242833052, "avg_output_tokens_per_task": 11989.828836424958, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7539281666666667, "accuracy_std_dev": 0.05140798989267677, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0019029528619528618, "avg_duration_per_task_ms": 665715.8501683502, "avg_output_tokens_per_task": 12898.684343434343, "last_run_timestamp": "2026-08-22T17:47:50.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7537277142857143, "accuracy_std_dev": 0.11897912112721323, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13752438961038962, "avg_duration_per_task_ms": 1748776.3001443, "avg_output_tokens_per_task": 48295.48701298701, "last_run_timestamp": "2026-10-03T17:21:47.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7504208749999999, "accuracy_std_dev": 0.14227753697829243, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.030610907196969697, "avg_duration_per_task_ms": 141563.8926767677, "avg_output_tokens_per_task": 23450.24873737374, "last_run_timestamp": "2026-10-03T16:48:25.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.7499064444444444, "accuracy_std_dev": 0.008289683281861894, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.01351321212121212, "avg_duration_per_task_ms": 98095.24747474748, "avg_output_tokens_per_task": 14852.885521885522, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.749158, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016407893939393937, "avg_duration_per_task_ms": 1120280.1313131314, "avg_output_tokens_per_task": 58254.84848484849, "last_run_timestamp": "2026-10-03T11:59:33.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7477631238816241, "accuracy_std_dev": 0.09701744760278624, "run_count": 8, "total_questions": 1453, "avg_cost_per_task_usd": 0.03234037852718513, "avg_duration_per_task_ms": 147355.79421885754, "avg_output_tokens_per_task": 28069.35856847901, "last_run_timestamp": "2026-10-03T17:57:43.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08105034848484848, "avg_duration_per_task_ms": 422601, "avg_output_tokens_per_task": 70367.68181818182, "last_run_timestamp": "2026-07-18T08:27:39.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016666974747474746, "avg_duration_per_task_ms": 1024552.0252525252, "avg_output_tokens_per_task": 59525.38383838384, "last_run_timestamp": "2026-10-03T09:35:36.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7474560693802035, "accuracy_std_dev": 0.030553086408774483, "run_count": 7, "total_questions": 1081, "avg_cost_per_task_usd": 0.007020049028677151, "avg_duration_per_task_ms": 981607.5957446808, "avg_output_tokens_per_task": 18100.610545790933, "last_run_timestamp": "2026-10-04T00:24:24.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7465211040462427, "accuracy_std_dev": 0.07408730878961323, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.20187897559409118, "avg_duration_per_task_ms": 872112.8529222865, "avg_output_tokens_per_task": 63704.67630057804, "last_run_timestamp": "2026-10-03T16:12:13.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7461931421319796, "accuracy_std_dev": 0.031200414988782142, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.004617404399323182, "avg_duration_per_task_ms": 792703.1945854484, "avg_output_tokens_per_task": 27906.86040609137, "last_run_timestamp": "2026-10-03T17:02:01.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7457915, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014859042929292928, "avg_duration_per_task_ms": 987926.6818181818, "avg_output_tokens_per_task": 52717.79797979798, "last_run_timestamp": "2026-10-03T10:50:00.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7457914444444445, "accuracy_std_dev": 0.00862087100666636, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.014793652076318742, "avg_duration_per_task_ms": 108312.31313131313, "avg_output_tokens_per_task": 14346.404601571268, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7455108333333333, "accuracy_std_dev": 0.020304992673532824, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.08322730471380473, "avg_duration_per_task_ms": 907303.9099326599, "avg_output_tokens_per_task": 23781.781144781144, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7451755736842105, "accuracy_std_dev": 0.011659500000000045, "run_count": 2, "total_questions": 380, "avg_cost_per_task_usd": 0.014937544736842105, "avg_duration_per_task_ms": 179152.7552631579, "avg_output_tokens_per_task": 54052.70263157895, "last_run_timestamp": "2026-10-03T09:41:31.910Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.744669, "accuracy_std_dev": 0.014698031999783739, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01758458585858586, "avg_duration_per_task_ms": 101220.53872053872, "avg_output_tokens_per_task": 6276.715488215488, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7446127, "accuracy_std_dev": 0.00984666853357015, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.006898398484848485, "avg_duration_per_task_ms": 165372.10808080807, "avg_output_tokens_per_task": 8017.161616161616, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.743897375, "accuracy_std_dev": 0.010561589427466657, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022802391414141414, "avg_duration_per_task_ms": 99703.82891414141, "avg_output_tokens_per_task": 15093.268308080807, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.7436867500000001, "accuracy_std_dev": 0.05045411551288456, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01589729419191919, "avg_duration_per_task_ms": 476456.76136363635, "avg_output_tokens_per_task": 15487.320707070707, "last_run_timestamp": "2026-10-04T00:29:23.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7426978294314383, "accuracy_std_dev": 0.00453846587516089, "run_count": 8, "total_questions": 1495, "avg_cost_per_task_usd": 0.008910557190635451, "avg_duration_per_task_ms": 136917.0508361204, "avg_output_tokens_per_task": 11277.847491638797, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.74242425, "accuracy_std_dev": 0.006838470877140611, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005376655303030303, "avg_duration_per_task_ms": 143698.49747474748, "avg_output_tokens_per_task": 8690.27840909091, "last_run_timestamp": "2026-10-03T17:05:50.656Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7422268568329717, "accuracy_std_dev": 0.051186502535695226, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.006795763557483733, "avg_duration_per_task_ms": 1042317.6109906002, "avg_output_tokens_per_task": 18262.2856109906, "last_run_timestamp": "2026-08-29T20:11:32.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7415824999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.019016893939393938, "avg_duration_per_task_ms": 33403.05808080808, "avg_output_tokens_per_task": 6109.055555555556, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.7407407777777778, "accuracy_std_dev": 0.010468392689719497, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.004919860830527497, "avg_duration_per_task_ms": 109862.1470258137, "avg_output_tokens_per_task": 8024.6543209876545, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7406781050847459, "accuracy_std_dev": 0.10340659429589846, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.006412678531073445, "avg_duration_per_task_ms": 534746.5333333333, "avg_output_tokens_per_task": 27485.332768361583, "last_run_timestamp": "2026-10-03T10:03:14.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7395381428571429, "accuracy_std_dev": 0.07707797586939116, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0066781443001443, "avg_duration_per_task_ms": 95940.86147186147, "avg_output_tokens_per_task": 16425.329725829724, "last_run_timestamp": "2026-10-03T16:54:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7392771671732522, "accuracy_std_dev": 0.07975374971554629, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.005161332320162108, "avg_duration_per_task_ms": 757299.1337386018, "avg_output_tokens_per_task": 34332.25075987842, "last_run_timestamp": "2026-09-26T18:04:35.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7388467499999999, "accuracy_std_dev": 0.07747406520692911, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01745328409090909, "avg_duration_per_task_ms": 2275674.0732323234, "avg_output_tokens_per_task": 17131.02398989899, "last_run_timestamp": "2026-08-22T18:24:57.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7387387252252252, "accuracy_std_dev": 0.018499983466208795, "run_count": 4, "total_questions": 444, "avg_cost_per_task_usd": 0.01493145045045045, "avg_duration_per_task_ms": 79529.70945945945, "avg_output_tokens_per_task": 8581.265765765766, "last_run_timestamp": "2026-10-04T00:00:36.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.738309, "accuracy_std_dev": 0.008136215650479735, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.059655349046015715, "avg_duration_per_task_ms": 125601.6632996633, "avg_output_tokens_per_task": 23853.966891133558, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7379703298337709, "accuracy_std_dev": 0.07144210650532616, "run_count": 7, "total_questions": 1143, "avg_cost_per_task_usd": 0.03304302712160979, "avg_duration_per_task_ms": 696117.6115485565, "avg_output_tokens_per_task": 35929.485564304465, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.736111, "accuracy_std_dev": null, "run_count": 1, "total_questions": 168, "avg_cost_per_task_usd": 0.05987852380952381, "avg_duration_per_task_ms": 1235915.3511904762, "avg_output_tokens_per_task": 62102.07738095238, "last_run_timestamp": "2026-07-18T08:29:05.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7345221716196138, "accuracy_std_dev": 0.016459468688265724, "run_count": 8, "total_questions": 1346, "avg_cost_per_task_usd": 0.006231184249628529, "avg_duration_per_task_ms": 219949.4487369985, "avg_output_tokens_per_task": 7708.209509658247, "last_run_timestamp": "2026-10-03T16:52:01.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7344877142857142, "accuracy_std_dev": 0.06178190798217846, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011764064935064936, "avg_duration_per_task_ms": 95375.22294372294, "avg_output_tokens_per_task": 5009.694083694084, "last_run_timestamp": "2026-10-03T10:46:59.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7316240615384616, "accuracy_std_dev": 0.012678999999999996, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.014848610256410256, "avg_duration_per_task_ms": 978212.5923076923, "avg_output_tokens_per_task": 52658.97692307692, "last_run_timestamp": "2026-10-03T10:14:05.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7316081564544914, "accuracy_std_dev": 0.016316209027987538, "run_count": 17, "total_questions": 3362, "avg_cost_per_task_usd": 0.006837803985722784, "avg_duration_per_task_ms": 102812.45925044616, "avg_output_tokens_per_task": 7135.6109458655565, "last_run_timestamp": "2026-10-03T16:40:17.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7313943448275863, "accuracy_std_dev": 0.013515629737139094, "run_count": 29, "total_questions": 5742, "avg_cost_per_task_usd": 0.2044269764890282, "avg_duration_per_task_ms": 243437.65552072448, "avg_output_tokens_per_task": 38184.69731800766, "last_run_timestamp": "2026-10-03T09:50:25.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7306398571428572, "accuracy_std_dev": 0.05292415038100302, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015631727272727273, "avg_duration_per_task_ms": 854213.9992784993, "avg_output_tokens_per_task": 42235.36435786436, "last_run_timestamp": "2026-10-03T18:22:40.169Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7297978999999999, "accuracy_std_dev": 0.016326424363282986, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.18559751616161616, "avg_duration_per_task_ms": 291435.0106060606, "avg_output_tokens_per_task": 36955.371717171714, "last_run_timestamp": "2026-10-03T09:40:02.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7286196, "accuracy_std_dev": 0.010123398502479267, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.19021829797979797, "avg_duration_per_task_ms": 241955.48484848486, "avg_output_tokens_per_task": 37852.89646464647, "last_run_timestamp": "2026-10-03T09:25:15.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.727903875, "accuracy_std_dev": 0.014553723049768914, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0046047815656565655, "avg_duration_per_task_ms": 179282.44696969696, "avg_output_tokens_per_task": 7813.462752525253, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7276937500000001, "accuracy_std_dev": 0.01898605081862735, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005756878787878787, "avg_duration_per_task_ms": 224182.07007575757, "avg_output_tokens_per_task": 7857.475378787879, "last_run_timestamp": "2026-10-03T17:25:00.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.7270113724137931, "accuracy_std_dev": 0.06592199999999998, "run_count": 2, "total_questions": 290, "avg_cost_per_task_usd": 0.06701382413793103, "avg_duration_per_task_ms": 911772.7965517241, "avg_output_tokens_per_task": 29657.59655172414, "last_run_timestamp": "2026-07-18T16:15:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.003979111111111111, "avg_duration_per_task_ms": 697805.6414141414, "avg_output_tokens_per_task": 30154.29292929293, "last_run_timestamp": "2026-07-18T16:28:34.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7261503333333333, "accuracy_std_dev": 0.004418546090690373, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.059926168350168355, "avg_duration_per_task_ms": 123042.70033670033, "avg_output_tokens_per_task": 23940.483164983165, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7260702857142858, "accuracy_std_dev": 0.011156764145758455, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006142652958152958, "avg_duration_per_task_ms": 382505.44805194804, "avg_output_tokens_per_task": 8147.619769119769, "last_run_timestamp": "2026-10-03T17:45:28.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7260101250000001, "accuracy_std_dev": 0.008051806325873379, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004004231691919192, "avg_duration_per_task_ms": 572174.6193181818, "avg_output_tokens_per_task": 8043.021464646465, "last_run_timestamp": "2026-10-03T17:33:20.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7255892500000001, "accuracy_std_dev": 0.028594658337484976, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008037106691919193, "avg_duration_per_task_ms": 265191.6982323232, "avg_output_tokens_per_task": 18033.603535353537, "last_run_timestamp": "2026-10-03T17:08:10.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7240554670050762, "accuracy_std_dev": 0.06016878242540359, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.014961366328257193, "avg_duration_per_task_ms": 1202663.8900169204, "avg_output_tokens_per_task": 38040.99492385787, "last_run_timestamp": "2026-10-03T17:20:21.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7234887554404145, "accuracy_std_dev": 0.009381891562472875, "run_count": 10, "total_questions": 1930, "avg_cost_per_task_usd": 0.18696787461139894, "avg_duration_per_task_ms": 349087.42279792746, "avg_output_tokens_per_task": 37203.45544041451, "last_run_timestamp": "2026-10-03T09:27:16.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7228535, "accuracy_std_dev": 0.01924905807435781, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0067263503787878785, "avg_duration_per_task_ms": 349070.6534090909, "avg_output_tokens_per_task": 26711.420454545456, "last_run_timestamp": "2026-10-03T17:40:21.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7225028666666667, "accuracy_std_dev": 0.012643758973589382, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.1973207321548822, "avg_duration_per_task_ms": 263893.9067340067, "avg_output_tokens_per_task": 36800.280303030304, "last_run_timestamp": "2026-10-03T09:27:45.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7222222500000001, "accuracy_std_dev": 0.0717218919764914, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02037232512626263, "avg_duration_per_task_ms": 1184721.3592171718, "avg_output_tokens_per_task": 14249.777777777777, "last_run_timestamp": "2026-10-03T16:57:23.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7221168064516129, "accuracy_std_dev": 0.11047255085087879, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.0044673187855787475, "avg_duration_per_task_ms": 283742.9405439595, "avg_output_tokens_per_task": 10988.973434535104, "last_run_timestamp": "2026-10-04T00:01:24.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.7205388571428571, "accuracy_std_dev": 0.045769541832435034, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006908489898989899, "avg_duration_per_task_ms": 128386.26262626263, "avg_output_tokens_per_task": 7669.774891774891, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.71923592494929, "accuracy_std_dev": 0.031713523289599956, "run_count": 5, "total_questions": 986, "avg_cost_per_task_usd": 0.01125353448275862, "avg_duration_per_task_ms": 528851.3123732251, "avg_output_tokens_per_task": 56006.78093306288, "last_run_timestamp": "2026-10-03T09:18:14.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7184345000000002, "accuracy_std_dev": 0.00575544305418794, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006676434343434343, "avg_duration_per_task_ms": 1198975.5896464647, "avg_output_tokens_per_task": 54074.54671717172, "last_run_timestamp": "2026-09-26T17:35:27.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7183463229974161, "accuracy_std_dev": 0.029264511720208542, "run_count": 6, "total_questions": 1161, "avg_cost_per_task_usd": 0.17074779758828595, "avg_duration_per_task_ms": 1524939.6339362618, "avg_output_tokens_per_task": 41593.74849267872, "last_run_timestamp": "2026-08-22T16:49:30.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.71485675, "accuracy_std_dev": 0.013413663339576564, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008089982323232324, "avg_duration_per_task_ms": 76524.31755050505, "avg_output_tokens_per_task": 10274.573232323231, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.712963, "accuracy_std_dev": 0.007926363478922754, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005945053872053872, "avg_duration_per_task_ms": 274691.60101010103, "avg_output_tokens_per_task": 7230.755892255892, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7110947790868927, "accuracy_std_dev": 0.09708503564674945, "run_count": 7, "total_questions": 1358, "avg_cost_per_task_usd": 0.009436205449189985, "avg_duration_per_task_ms": 507448.52209131076, "avg_output_tokens_per_task": 32737.85714285714, "last_run_timestamp": "2026-10-03T17:30:20.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.7108066232876712, "accuracy_std_dev": 0.016923430604933478, "run_count": 5, "total_questions": 876, "avg_cost_per_task_usd": 0.06916225000000001, "avg_duration_per_task_ms": 1230338.2773972603, "avg_output_tokens_per_task": 23160.62100456621, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7107282190293742, "accuracy_std_dev": 0.01626806742325285, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.010758521711366539, "avg_duration_per_task_ms": 103410.69029374202, "avg_output_tokens_per_task": 9911.577266922095, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7106482500000001, "accuracy_std_dev": 0.007960486601803957, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.015934953282828283, "avg_duration_per_task_ms": 96234.6571969697, "avg_output_tokens_per_task": 10542.059343434343, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7094277999999999, "accuracy_std_dev": 0.0745134320251054, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007971388888888889, "avg_duration_per_task_ms": 178465.76363636364, "avg_output_tokens_per_task": 7431.1, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.709385625, "accuracy_std_dev": 0.009446333705961032, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00881291351010101, "avg_duration_per_task_ms": 137334.60416666666, "avg_output_tokens_per_task": 5712.794191919192, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.709175125, "accuracy_std_dev": 0.01063905747044237, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.011125260732323233, "avg_duration_per_task_ms": 96740.25820707071, "avg_output_tokens_per_task": 10516.064393939394, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.70883299486008, "accuracy_std_dev": 0.036241367349453366, "run_count": 9, "total_questions": 1751, "avg_cost_per_task_usd": 0.010877087378640777, "avg_duration_per_task_ms": 281941.7778412336, "avg_output_tokens_per_task": 47824.729868646486, "last_run_timestamp": "2026-10-03T09:36:23.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7070005146036162, "accuracy_std_dev": 0.06882289073774799, "run_count": 4, "total_questions": 719, "avg_cost_per_task_usd": 0.07411282058414463, "avg_duration_per_task_ms": 612487.7343532684, "avg_output_tokens_per_task": 29410.726008344922, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.7062016019933554, "accuracy_std_dev": 0.2586002901946361, "run_count": 8, "total_questions": 1505, "avg_cost_per_task_usd": 0.025144299667774088, "avg_duration_per_task_ms": 338978.96146179404, "avg_output_tokens_per_task": 15167.856478405316, "last_run_timestamp": "2026-10-04T00:21:14.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.704545375, "accuracy_std_dev": 0.008501245907769926, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006021614898989899, "avg_duration_per_task_ms": 108658.90909090909, "avg_output_tokens_per_task": 5588.438762626262, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7039041742125984, "accuracy_std_dev": 0.06488902055802347, "run_count": 7, "total_questions": 1016, "avg_cost_per_task_usd": 0.03703799311023622, "avg_duration_per_task_ms": 230051.07480314962, "avg_output_tokens_per_task": 14035.937992125984, "last_run_timestamp": "2026-08-29T19:20:11.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7037214026717559, "accuracy_std_dev": 0.050640957870974806, "run_count": 16, "total_questions": 3144, "avg_cost_per_task_usd": 0.008582292620865142, "avg_duration_per_task_ms": 446340.70642493636, "avg_output_tokens_per_task": 25644.754770992367, "last_run_timestamp": "2026-10-03T17:44:46.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7034933750000001, "accuracy_std_dev": 0.013610377795431516, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.011181943813131312, "avg_duration_per_task_ms": 198847.17866161617, "avg_output_tokens_per_task": 5822.409722222223, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.70328275, "accuracy_std_dev": 0.02153875190644758, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0019388005050505052, "avg_duration_per_task_ms": 636092.8409090909, "avg_output_tokens_per_task": 14000.35101010101, "last_run_timestamp": "2026-09-05T08:28:07.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7024411666666667, "accuracy_std_dev": 0.14218853509503576, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013173934343434343, "avg_duration_per_task_ms": 786438.8409090909, "avg_output_tokens_per_task": 27569.127104377105, "last_run_timestamp": "2026-09-22T02:20:03.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Wafer", "accuracy": 0.70202, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.012498363636363634, "avg_duration_per_task_ms": 411305.4646464646, "avg_output_tokens_per_task": 31035.28787878788, "last_run_timestamp": "2026-07-18T08:04:42.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.7015995, "accuracy_std_dev": 0.023308248459504657, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.011333077020202021, "avg_duration_per_task_ms": 63982.55744949495, "avg_output_tokens_per_task": 9378.460858585859, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7013888749999999, "accuracy_std_dev": 0.009189646775549915, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010544860479797979, "avg_duration_per_task_ms": 145490.4551767677, "avg_output_tokens_per_task": 9519.845328282829, "last_run_timestamp": "2026-10-03T16:56:42.844Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6994219060693642, "accuracy_std_dev": 0.028501362281769377, "run_count": 6, "total_questions": 692, "avg_cost_per_task_usd": 0.010150841040462426, "avg_duration_per_task_ms": 46976.1950867052, "avg_output_tokens_per_task": 6570.052023121388, "last_run_timestamp": "2026-10-04T00:12:48.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6960783363171356, "accuracy_std_dev": 0.1309593611346665, "run_count": 5, "total_questions": 782, "avg_cost_per_task_usd": 0.0047808593350383635, "avg_duration_per_task_ms": 587688.8618925831, "avg_output_tokens_per_task": 25488.57928388747, "last_run_timestamp": "2026-10-03T09:54:28.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.6956243088803089, "accuracy_std_dev": 0.0895741115626294, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.12983634813384812, "avg_duration_per_task_ms": 1082525.285070785, "avg_output_tokens_per_task": 70424.97168597169, "last_run_timestamp": "2026-10-03T16:56:56.694Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.694865375, "accuracy_std_dev": 0.01804853225291118, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008836173611111112, "avg_duration_per_task_ms": 277270.93055555556, "avg_output_tokens_per_task": 6352.661616161616, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6931105699373695, "accuracy_std_dev": 0.08497842021164353, "run_count": 6, "total_questions": 958, "avg_cost_per_task_usd": 0.0069657160751565765, "avg_duration_per_task_ms": 481855.8267223382, "avg_output_tokens_per_task": 32574.692066805845, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6929713750000001, "accuracy_std_dev": 0.008745132416628961, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005507261994949495, "avg_duration_per_task_ms": 385008.58585858584, "avg_output_tokens_per_task": 6073.715277777777, "last_run_timestamp": "2026-10-03T17:41:44.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6926927837837837, "accuracy_std_dev": 0.02593753677273029, "run_count": 7, "total_questions": 1332, "avg_cost_per_task_usd": 0.009898264264264264, "avg_duration_per_task_ms": 376272.99399399396, "avg_output_tokens_per_task": 45270.37087087087, "last_run_timestamp": "2026-10-03T09:53:13.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.6896746666666668, "accuracy_std_dev": 0.04960523518294776, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04329939057239058, "avg_duration_per_task_ms": 1345214.5555555555, "avg_output_tokens_per_task": 31899.07744107744, "last_run_timestamp": "2026-08-22T16:20:35.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.685185, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007298616161616161, "avg_duration_per_task_ms": 578052.9191919192, "avg_output_tokens_per_task": 17923.358585858587, "last_run_timestamp": "2026-07-18T08:44:31.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.684238375, "accuracy_std_dev": 0.042521690017970536, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01436286111111111, "avg_duration_per_task_ms": 255009.83333333334, "avg_output_tokens_per_task": 7755.36553030303, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.681700944587629, "accuracy_std_dev": 0.10151777573533613, "run_count": 4, "total_questions": 776, "avg_cost_per_task_usd": 0.061980561855670094, "avg_duration_per_task_ms": 552656.2654639175, "avg_output_tokens_per_task": 25817.090206185567, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6815885046931408, "accuracy_std_dev": 0.03569859904757024, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.005665397111913357, "avg_duration_per_task_ms": 406427.05487364624, "avg_output_tokens_per_task": 15523.718411552347, "last_run_timestamp": "2026-10-03T17:31:07.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6762865714285715, "accuracy_std_dev": 0.05037246428033344, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008260983405483407, "avg_duration_per_task_ms": 263120.9155844156, "avg_output_tokens_per_task": 42075.49062049062, "last_run_timestamp": "2026-10-03T09:28:39.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.675926, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023789772727272726, "avg_duration_per_task_ms": 1250422.4444444445, "avg_output_tokens_per_task": 14190.916666666666, "last_run_timestamp": "2026-07-18T09:58:41.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6757730775095298, "accuracy_std_dev": 0.020105082568893876, "run_count": 7, "total_questions": 787, "avg_cost_per_task_usd": 0.005445952986022871, "avg_duration_per_task_ms": 625853.6467598475, "avg_output_tokens_per_task": 36252.05082592122, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6757497120822621, "accuracy_std_dev": 0.20642627550360443, "run_count": 10, "total_questions": 1945, "avg_cost_per_task_usd": 0.014975519280205657, "avg_duration_per_task_ms": 371086.62262210797, "avg_output_tokens_per_task": 18143.37994858612, "last_run_timestamp": "2026-09-26T08:53:03.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6750842, "accuracy_std_dev": 0.023641210809939493, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005652539393939394, "avg_duration_per_task_ms": 500294.62828282826, "avg_output_tokens_per_task": 16033.422222222222, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.6694725555555555, "accuracy_std_dev": 0.032119090204740325, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0024531571268237937, "avg_duration_per_task_ms": 211506.06902356903, "avg_output_tokens_per_task": 5166.5179573512905, "last_run_timestamp": "2026-10-03T16:12:07.496Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.6688176666666668, "accuracy_std_dev": 0.09003633453223205, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0024410033670033675, "avg_duration_per_task_ms": 222109.77160493826, "avg_output_tokens_per_task": 5583.682940516273, "last_run_timestamp": "2026-10-03T16:12:35.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.6673561406411582, "accuracy_std_dev": 0.17106562595518718, "run_count": 5, "total_questions": 967, "avg_cost_per_task_usd": 0.07843655946225439, "avg_duration_per_task_ms": 1463737.4043433298, "avg_output_tokens_per_task": 37688.086866597725, "last_run_timestamp": "2026-08-22T17:02:13.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.6649835000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009143497474747474, "avg_duration_per_task_ms": 686922.851010101, "avg_output_tokens_per_task": 70113.30555555556, "last_run_timestamp": "2026-07-18T17:04:25.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6637464899415964, "accuracy_std_dev": 0.013487143446978517, "run_count": 8, "total_questions": 1541, "avg_cost_per_task_usd": 0.007471570408825438, "avg_duration_per_task_ms": 501530.87605451007, "avg_output_tokens_per_task": 53289.73977936405, "last_run_timestamp": "2026-10-03T18:05:00.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6622475000000001, "accuracy_std_dev": 0.023522840681983983, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.028767857323232324, "avg_duration_per_task_ms": 847663.7582070708, "avg_output_tokens_per_task": 19263.888257575756, "last_run_timestamp": "2026-10-03T16:57:14.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6621005671232876, "accuracy_std_dev": 0.014348054525498073, "run_count": 3, "total_questions": 365, "avg_cost_per_task_usd": 0.004233909589041096, "avg_duration_per_task_ms": 723664.6465753425, "avg_output_tokens_per_task": 42200.62191780822, "last_run_timestamp": "2026-10-03T19:04:05.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.6616163333333333, "accuracy_std_dev": 0.0111670002935236, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0024947028619528616, "avg_duration_per_task_ms": 94165.7398989899, "avg_output_tokens_per_task": 2312.0664983164984, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6607744999999999, "accuracy_std_dev": 0.011572094721786573, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015565636363636365, "avg_duration_per_task_ms": 14159.460858585859, "avg_output_tokens_per_task": 1751.3977272727273, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6606341666666665, "accuracy_std_dev": 0.013370433711697206, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.00517740867003367, "avg_duration_per_task_ms": 106914.1734006734, "avg_output_tokens_per_task": 9753.443602693602, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6599325714285714, "accuracy_std_dev": 0.010532623440084252, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0027436955266955266, "avg_duration_per_task_ms": 79836.26262626263, "avg_output_tokens_per_task": 2910.7698412698414, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6596921428571428, "accuracy_std_dev": 0.011884572910513284, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0022321969696969695, "avg_duration_per_task_ms": 222919.87445887446, "avg_output_tokens_per_task": 3156.7669552669554, "last_run_timestamp": "2026-08-29T18:47:21.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6596062028241334, "accuracy_std_dev": 0.016648273731831805, "run_count": 4, "total_questions": 779, "avg_cost_per_task_usd": 0.011759379974326058, "avg_duration_per_task_ms": 1526360.2888318356, "avg_output_tokens_per_task": 50819.251604621306, "last_run_timestamp": "2026-10-03T17:41:49.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6571429371428571, "accuracy_std_dev": 0.01564259566443798, "run_count": 16, "total_questions": 3150, "avg_cost_per_task_usd": 0.009064113015873019, "avg_duration_per_task_ms": 336807.49333333335, "avg_output_tokens_per_task": 60106.63492063492, "last_run_timestamp": "2026-10-03T17:56:49.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.654882, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01704559090909091, "avg_duration_per_task_ms": 1937600.9292929294, "avg_output_tokens_per_task": 68071.47474747474, "last_run_timestamp": "2026-10-01T08:41:43.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.6539546483050847, "accuracy_std_dev": 0.06028980821738355, "run_count": 3, "total_questions": 236, "avg_cost_per_task_usd": 0.09281549576271186, "avg_duration_per_task_ms": 1220256.3305084745, "avg_output_tokens_per_task": 32723.118644067796, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6512783352272727, "accuracy_std_dev": 0.07301920016707937, "run_count": 5, "total_questions": 704, "avg_cost_per_task_usd": 0.06126106249999999, "avg_duration_per_task_ms": 1570715.0482954546, "avg_output_tokens_per_task": 22881.873579545456, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.6509389830985916, "accuracy_std_dev": 0.10781834547167748, "run_count": 4, "total_questions": 710, "avg_cost_per_task_usd": 0.0075063676056338026, "avg_duration_per_task_ms": 162639.05070422534, "avg_output_tokens_per_task": 9309.304225352113, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.650363823580786, "accuracy_std_dev": 0.007103203700131033, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.0036315545851528385, "avg_duration_per_task_ms": 58503.03493449782, "avg_output_tokens_per_task": 2202.661135371179, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.011195005920300148, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004075954545454546, "avg_duration_per_task_ms": 28668.510101010103, "avg_output_tokens_per_task": 2354.649831649832, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6487794999999998, "accuracy_std_dev": 0.12844600199597497, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6039836092171716, "avg_duration_per_task_ms": 625325.3693181818, "avg_output_tokens_per_task": 40216.142676767675, "last_run_timestamp": "2026-10-03T10:37:00.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6478542063492063, "accuracy_std_dev": 0.03133768881430511, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.025561564373897706, "avg_duration_per_task_ms": 917973.6243386243, "avg_output_tokens_per_task": 30424.42680776014, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6478421818181818, "accuracy_std_dev": 0.01794361558690384, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.00699533930211203, "avg_duration_per_task_ms": 234218.77502295683, "avg_output_tokens_per_task": 11266.905417814509, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6476671428571429, "accuracy_std_dev": 0.01801337100543905, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007287767676767677, "avg_duration_per_task_ms": 701126.4617604618, "avg_output_tokens_per_task": 56538.52525252525, "last_run_timestamp": "2026-10-03T18:53:13.795Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6474267142857143, "accuracy_std_dev": 0.08020884466675425, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005957715007215008, "avg_duration_per_task_ms": 632251.2438672439, "avg_output_tokens_per_task": 21165.58008658009, "last_run_timestamp": "2026-10-03T17:57:14.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6473065000000001, "accuracy_std_dev": 0.009707729665065862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00413114393939394, "avg_duration_per_task_ms": 27377.72601010101, "avg_output_tokens_per_task": 2388.9305555555557, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.6471861428571428, "accuracy_std_dev": 0.010488726920277937, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0025087236652236652, "avg_duration_per_task_ms": 65942.57142857143, "avg_output_tokens_per_task": 2709.98556998557, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.64646475, "accuracy_std_dev": 0.02185317805692116, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007503073232323232, "avg_duration_per_task_ms": 814987.6553030303, "avg_output_tokens_per_task": 52529.47348484849, "last_run_timestamp": "2026-10-03T20:08:16.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6464646666666667, "accuracy_std_dev": 0.013537798302858865, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015368191919191919, "avg_duration_per_task_ms": 14060.520202020201, "avg_output_tokens_per_task": 1727.3535353535353, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6459835714285715, "accuracy_std_dev": 0.011408025080832253, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0032850678210678214, "avg_duration_per_task_ms": 40715.905483405484, "avg_output_tokens_per_task": 3121.487012987013, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6450217142857143, "accuracy_std_dev": 0.013022733909966664, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0023176284271284268, "avg_duration_per_task_ms": 69183.5432900433, "avg_output_tokens_per_task": 2807.774891774892, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.6447570300850229, "accuracy_std_dev": 0.0798361906300707, "run_count": 9, "total_questions": 1529, "avg_cost_per_task_usd": 0.02925614519293656, "avg_duration_per_task_ms": 627719.6899934597, "avg_output_tokens_per_task": 30710.851536952257, "last_run_timestamp": "2026-10-03T17:54:48.467Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.6446141374045801, "accuracy_std_dev": 0.005257999999999985, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.0025083384223918577, "avg_duration_per_task_ms": 104571.39694656488, "avg_output_tokens_per_task": 8089.860050890586, "last_run_timestamp": "2026-07-18T08:33:31.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.6443544137931034, "accuracy_std_dev": 0.06155566230851553, "run_count": 5, "total_questions": 986, "avg_cost_per_task_usd": 0.04049388945233266, "avg_duration_per_task_ms": 420330.3397565923, "avg_output_tokens_per_task": 20151.943204868156, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6439909795918367, "accuracy_std_dev": 0.019637275869637497, "run_count": 9, "total_questions": 1764, "avg_cost_per_task_usd": 0.01045445804988662, "avg_duration_per_task_ms": 693718.4659863946, "avg_output_tokens_per_task": 52902.34467120181, "last_run_timestamp": "2026-10-03T20:22:45.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6438963244781784, "accuracy_std_dev": 0.010816729929113277, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.0026027520556609744, "avg_duration_per_task_ms": 140999.67488931055, "avg_output_tokens_per_task": 9253.603415559772, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.643659, "accuracy_std_dev": 0.015752669318774726, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015761657407407408, "avg_duration_per_task_ms": 18961.11531986532, "avg_output_tokens_per_task": 1683.8872053872053, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.640783, "accuracy_std_dev": 0.01270844444650878, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004579357954545454, "avg_duration_per_task_ms": 73293.53661616161, "avg_output_tokens_per_task": 8611.67803030303, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6405723333333334, "accuracy_std_dev": 0.0030349496792460334, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004210166666666667, "avg_duration_per_task_ms": 30431.28367003367, "avg_output_tokens_per_task": 2308.9638047138046, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.6404040121212122, "accuracy_std_dev": 0.11145785266385451, "run_count": 8, "total_questions": 1485, "avg_cost_per_task_usd": 0.09477065723905725, "avg_duration_per_task_ms": 760875.5326599326, "avg_output_tokens_per_task": 29720.403367003368, "last_run_timestamp": "2026-10-03T16:55:14.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6386253795066414, "accuracy_std_dev": 0.016118987931551414, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.002519017710309931, "avg_duration_per_task_ms": 220560.1669829222, "avg_output_tokens_per_task": 8172.611638203669, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6384837407407408, "accuracy_std_dev": 0.1301056437821581, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.6548667364384586, "avg_duration_per_task_ms": 493199.0927796483, "avg_output_tokens_per_task": 40750.34081556304, "last_run_timestamp": "2026-10-03T10:04:02.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6381495969581749, "accuracy_std_dev": 0.013726909852890242, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.003178458808618504, "avg_duration_per_task_ms": 206354.68187579213, "avg_output_tokens_per_task": 10325.579847908744, "last_run_timestamp": "2026-10-03T16:41:22.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6376733333333334, "accuracy_std_dev": 0.015547005006324102, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0016130179573512905, "avg_duration_per_task_ms": 106654.77497194163, "avg_output_tokens_per_task": 8152.802469135802, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6374935167427701, "accuracy_std_dev": 0.023026873072564586, "run_count": 8, "total_questions": 1314, "avg_cost_per_task_usd": 0.016053601217656012, "avg_duration_per_task_ms": 96215.6704718417, "avg_output_tokens_per_task": 53290.504566210046, "last_run_timestamp": "2026-10-03T17:21:59.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6371463734317343, "accuracy_std_dev": 0.2220532628164637, "run_count": 8, "total_questions": 1355, "avg_cost_per_task_usd": 0.007668025830258302, "avg_duration_per_task_ms": 311815.4856088561, "avg_output_tokens_per_task": 12706.190405904059, "last_run_timestamp": "2026-10-03T16:57:17.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6368643521865888, "accuracy_std_dev": 0.13454240742991402, "run_count": 26, "total_questions": 5145, "avg_cost_per_task_usd": 0.6621504369290573, "avg_duration_per_task_ms": 598729.5551020408, "avg_output_tokens_per_task": 41288.28415937804, "last_run_timestamp": "2026-10-03T10:34:21.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.636738, "accuracy_std_dev": 0.1325038277837034, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.6152635606060606, "avg_duration_per_task_ms": 631219.6868686868, "avg_output_tokens_per_task": 40865.88888888889, "last_run_timestamp": "2026-10-03T09:11:52.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.635522, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013100151515151516, "avg_duration_per_task_ms": 444234.55555555556, "avg_output_tokens_per_task": 31287.63888888889, "last_run_timestamp": "2026-07-18T16:20:00.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6352981489141675, "accuracy_std_dev": 0.02103570346434842, "run_count": 5, "total_questions": 967, "avg_cost_per_task_usd": 0.006571570837642193, "avg_duration_per_task_ms": 890942.791106515, "avg_output_tokens_per_task": 36274.74457083764, "last_run_timestamp": "2026-10-03T08:12:46.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6352257765567765, "accuracy_std_dev": 0.022072905759475025, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.013415796703296703, "avg_duration_per_task_ms": 548616.1373626373, "avg_output_tokens_per_task": 46655.3021978022, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6349608333333333, "accuracy_std_dev": 0.0326935856217325, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.006947168350168351, "avg_duration_per_task_ms": 113160.32070707071, "avg_output_tokens_per_task": 9692.425505050505, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6337581592356688, "accuracy_std_dev": 0.004703999999999986, "run_count": 2, "total_questions": 314, "avg_cost_per_task_usd": 0.002880643312101911, "avg_duration_per_task_ms": 145451.6560509554, "avg_output_tokens_per_task": 9370.200636942674, "last_run_timestamp": "2026-10-03T17:25:22.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6337451904761905, "accuracy_std_dev": 0.08567913487347233, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.03312850793650794, "avg_duration_per_task_ms": 835240.3333333334, "avg_output_tokens_per_task": 32942.40211640212, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.633745, "accuracy_std_dev": 0.1400560587510904, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.6191078804713805, "avg_duration_per_task_ms": 634861.303030303, "avg_output_tokens_per_task": 41204.60381593715, "last_run_timestamp": "2026-10-03T10:40:39.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.6326063626168223, "accuracy_std_dev": 0.02740974656739623, "run_count": 9, "total_questions": 1605, "avg_cost_per_task_usd": 0.022848554517133954, "avg_duration_per_task_ms": 1040840.2074766355, "avg_output_tokens_per_task": 95001.86853582555, "last_run_timestamp": "2026-10-03T17:27:37.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6325426993295513, "accuracy_std_dev": 0.010534965676735725, "run_count": 10, "total_questions": 1939, "avg_cost_per_task_usd": 0.01766013512119649, "avg_duration_per_task_ms": 478451.96183599794, "avg_output_tokens_per_task": 88282.73388344508, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.6317296185567011, "accuracy_std_dev": 0.03997713513997718, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.02984942783505154, "avg_duration_per_task_ms": 817292.206185567, "avg_output_tokens_per_task": 36396.65120274914, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6308931798152096, "accuracy_std_dev": 0.018213636160511713, "run_count": 8, "total_questions": 1407, "avg_cost_per_task_usd": 0.007495727078891258, "avg_duration_per_task_ms": 35794.715707178395, "avg_output_tokens_per_task": 12413.633262260128, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6305648888888888, "accuracy_std_dev": 0.016837183193056476, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0016988282828282828, "avg_duration_per_task_ms": 204946.65937149271, "avg_output_tokens_per_task": 9301.569023569024, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6293489999999999, "accuracy_std_dev": 0.008199388066597833, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0011419814814814814, "avg_duration_per_task_ms": 334863.3063973064, "avg_output_tokens_per_task": 11634.42760942761, "last_run_timestamp": "2026-10-03T17:38:50.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.628788, "accuracy_std_dev": 0.01430999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017457550505050507, "avg_duration_per_task_ms": 1341424.7676767677, "avg_output_tokens_per_task": 31206.515151515152, "last_run_timestamp": "2026-07-18T17:46:02.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6285795064570466, "accuracy_std_dev": 0.020919173358428876, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01718929421673217, "avg_duration_per_task_ms": 631165.4817518248, "avg_output_tokens_per_task": 83912.1010668164, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6284272857142859, "accuracy_std_dev": 0.0807098788319077, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011130141414141415, "avg_duration_per_task_ms": 1152637.8816738818, "avg_output_tokens_per_task": 32863.90331890332, "last_run_timestamp": "2026-10-03T17:46:52.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.626793113785558, "accuracy_std_dev": 0.019460388437090578, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.014277733041575492, "avg_duration_per_task_ms": 690347.9810357403, "avg_output_tokens_per_task": 14917.212253829322, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.6249264761904761, "accuracy_std_dev": 0.06315233322688879, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.029899758377425045, "avg_duration_per_task_ms": 776085.4091710758, "avg_output_tokens_per_task": 36770.42328042328, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6212123333333334, "accuracy_std_dev": 0.009622003960829715, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00826737037037037, "avg_duration_per_task_ms": 370134.4276094276, "avg_output_tokens_per_task": 15814.858585858587, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6164022857142857, "accuracy_std_dev": 0.022360138497617886, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0054965389610389616, "avg_duration_per_task_ms": 603772.5, "avg_output_tokens_per_task": 18275.68831168831, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6162989117147707, "accuracy_std_dev": 0.004859400671779274, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.0033007164685908323, "avg_duration_per_task_ms": 414472.32767402375, "avg_output_tokens_per_task": 13345.62818336163, "last_run_timestamp": "2026-10-01T16:36:31.937Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6132539090909092, "accuracy_std_dev": 0.1299958112139524, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.6145130344352616, "avg_duration_per_task_ms": 628297.9467401286, "avg_output_tokens_per_task": 40850.65381083563, "last_run_timestamp": "2026-10-03T10:13:27.517Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6131824185936763, "accuracy_std_dev": 0.020395407276319392, "run_count": 11, "total_questions": 2119, "avg_cost_per_task_usd": 0.008309551675318546, "avg_duration_per_task_ms": 519028.40207645117, "avg_output_tokens_per_task": 13645.85040113261, "last_run_timestamp": "2026-10-03T10:33:57.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6107830459317585, "accuracy_std_dev": 0.004553237055930632, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.004046526902887138, "avg_duration_per_task_ms": 40209.83464566929, "avg_output_tokens_per_task": 2149.975721784777, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "NextBit", "accuracy": 0.6102695, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00920193181818182, "avg_duration_per_task_ms": 945597.9292929292, "avg_output_tokens_per_task": 16723.31818181818, "last_run_timestamp": "2026-07-18T08:03:56.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.609638125, "accuracy_std_dev": 0.00961364604920396, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.003643919823232323, "avg_duration_per_task_ms": 34782.93308080808, "avg_output_tokens_per_task": 2151.7373737373737, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6090626312364424, "accuracy_std_dev": 0.009643076431368988, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.008417777295733913, "avg_duration_per_task_ms": 272268.59725234995, "avg_output_tokens_per_task": 17483.89949385394, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6081999956331878, "accuracy_std_dev": 0.04268956350879486, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.023787051673944692, "avg_duration_per_task_ms": 1027252.8209606986, "avg_output_tokens_per_task": 62021.267831149926, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.606622, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007795220538720538, "avg_duration_per_task_ms": 300972.74074074073, "avg_output_tokens_per_task": 15395.011784511784, "last_run_timestamp": "2026-08-01T18:16:00.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6063702713347922, "accuracy_std_dev": 0.021503606331620406, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.06956871261852661, "avg_duration_per_task_ms": 1033277.1312910285, "avg_output_tokens_per_task": 26710.033552151715, "last_run_timestamp": "2026-08-30T00:22:07.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.6060606666666666, "accuracy_std_dev": 0.0013745720142000956, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008301892255892255, "avg_duration_per_task_ms": 153838.79292929292, "avg_output_tokens_per_task": 15775.223905723906, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.6060605, "accuracy_std_dev": 0.001683499999999949, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004138252525252526, "avg_duration_per_task_ms": 50192.44191919192, "avg_output_tokens_per_task": 2217.2070707070707, "last_run_timestamp": "2026-07-18T08:26:39.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.60437725, "accuracy_std_dev": 0.004908375259441744, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.011036202651515153, "avg_duration_per_task_ms": 34065.54861111111, "avg_output_tokens_per_task": 2102.629419191919, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.604166875, "accuracy_std_dev": 0.006651157426296166, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.003487729797979798, "avg_duration_per_task_ms": 53432.55744949495, "avg_output_tokens_per_task": 2119.665404040404, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6041191250947687, "accuracy_std_dev": 0.014817500782134993, "run_count": 7, "total_questions": 1319, "avg_cost_per_task_usd": 0.0021458051554207732, "avg_duration_per_task_ms": 47625.93252463988, "avg_output_tokens_per_task": 2007.382107657316, "last_run_timestamp": "2026-10-03T16:40:33.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.6004493333333333, "accuracy_std_dev": 0.0031744380430067634, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014951090909090909, "avg_duration_per_task_ms": 287967.1666666667, "avg_output_tokens_per_task": 12268.760942760942, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.5982746250000001, "accuracy_std_dev": 0.011782541692452221, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0032721849747474746, "avg_duration_per_task_ms": 64540.08964646464, "avg_output_tokens_per_task": 2086.2304292929293, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5929293999999999, "accuracy_std_dev": 0.011437975198434383, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0030511959595959596, "avg_duration_per_task_ms": 648893.6383838383, "avg_output_tokens_per_task": 13498.415151515152, "last_run_timestamp": "2026-10-02T16:16:46.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.591751, "accuracy_std_dev": 0.006627755628164546, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017204867003367003, "avg_duration_per_task_ms": 118615.4696969697, "avg_output_tokens_per_task": 42873.59259259259, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.58642, "accuracy_std_dev": null, "run_count": 1, "total_questions": 162, "avg_cost_per_task_usd": 0.002223469135802469, "avg_duration_per_task_ms": 407240.81481481483, "avg_output_tokens_per_task": 10928.734567901234, "last_run_timestamp": "2026-10-03T09:17:20.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5858372098049551, "accuracy_std_dev": 0.06404439817310623, "run_count": 11, "total_questions": 1897, "avg_cost_per_task_usd": 0.008970768581971532, "avg_duration_per_task_ms": 524600.9272535583, "avg_output_tokens_per_task": 44661.65946230891, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5839199654510556, "accuracy_std_dev": 0.014881331030858787, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.006653753678822777, "avg_duration_per_task_ms": 744303.1906589891, "avg_output_tokens_per_task": 44099.619321817016, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5797561250000001, "accuracy_std_dev": 0.01826692000884044, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010468239898989899, "avg_duration_per_task_ms": 11933.318813131313, "avg_output_tokens_per_task": 13503.182449494949, "last_run_timestamp": "2026-10-03T17:02:12.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.5767120715096482, "accuracy_std_dev": 0.20359371905450435, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.023737154370034052, "avg_duration_per_task_ms": 751908.8337116913, "avg_output_tokens_per_task": 33435.202610669694, "last_run_timestamp": "2026-10-03T17:20:14.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5709477142857142, "accuracy_std_dev": 0.027047512796739667, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004546116883116884, "avg_duration_per_task_ms": 351634.9336219336, "avg_output_tokens_per_task": 16008.746753246753, "last_run_timestamp": "2026-08-29T19:16:45.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5686779050064185, "accuracy_std_dev": 0.0536708857668662, "run_count": 6, "total_questions": 779, "avg_cost_per_task_usd": 0.053176206675224644, "avg_duration_per_task_ms": 695387.1527599486, "avg_output_tokens_per_task": 25450.709884467266, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5676931173020527, "accuracy_std_dev": 0.02908231088142895, "run_count": 7, "total_questions": 682, "avg_cost_per_task_usd": 0.032412470674486805, "avg_duration_per_task_ms": 57640.85923753666, "avg_output_tokens_per_task": 6927.941348973607, "last_run_timestamp": "2026-10-04T00:00:34.891Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.5651467231270357, "accuracy_std_dev": 0.004022999999999999, "run_count": 2, "total_questions": 307, "avg_cost_per_task_usd": 0.003757250814332247, "avg_duration_per_task_ms": 2542703.263843648, "avg_output_tokens_per_task": 34539.9674267101, "last_run_timestamp": "2026-07-18T16:00:09.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5637339187817259, "accuracy_std_dev": 0.03764953294383463, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.025736050761421318, "avg_duration_per_task_ms": 1231284.9746192894, "avg_output_tokens_per_task": 65120.38747884941, "last_run_timestamp": "2026-08-30T00:49:28.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5505053333333333, "accuracy_std_dev": 0.016890890559378122, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.017397131313131312, "avg_duration_per_task_ms": 129348.1271043771, "avg_output_tokens_per_task": 43400.656565656565, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5488216666666668, "accuracy_std_dev": 0.006298902038362641, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0171891531986532, "avg_duration_per_task_ms": 121387.12121212122, "avg_output_tokens_per_task": 42892.78114478115, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.5482828260135134, "accuracy_std_dev": 0.09041648367539086, "run_count": 6, "total_questions": 1184, "avg_cost_per_task_usd": 0.02739165118243243, "avg_duration_per_task_ms": 2548856.001689189, "avg_output_tokens_per_task": 22585.01097972973, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5468087361702126, "accuracy_std_dev": 0.04498858864739341, "run_count": 8, "total_questions": 1410, "avg_cost_per_task_usd": 0.016162113475177306, "avg_duration_per_task_ms": 706288.4482269504, "avg_output_tokens_per_task": 40872.247517730495, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0071059848484848484, "avg_duration_per_task_ms": 292481.80303030304, "avg_output_tokens_per_task": 36717.48484848485, "last_run_timestamp": "2026-07-11T15:34:14.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5409653333333333, "accuracy_std_dev": 0.009356358206529358, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0018076902356902358, "avg_duration_per_task_ms": 84884.97474747474, "avg_output_tokens_per_task": 1770.5521885521885, "last_run_timestamp": "2026-10-01T19:33:41.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.5396825971428572, "accuracy_std_dev": 0.025001612450040917, "run_count": 7, "total_questions": 1050, "avg_cost_per_task_usd": 0.09786349714285715, "avg_duration_per_task_ms": 941441.8342857143, "avg_output_tokens_per_task": 27874.138095238097, "last_run_timestamp": "2026-08-29T18:52:09.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.5342313333333333, "accuracy_std_dev": 0.002099515235053612, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005432905723905724, "avg_duration_per_task_ms": 120646.2558922559, "avg_output_tokens_per_task": 11731.540404040405, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.5331086666666667, "accuracy_std_dev": 0.05754091454106565, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005119237373737374, "avg_duration_per_task_ms": 101325.84511784512, "avg_output_tokens_per_task": 5420.193602693603, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.5309734740834386, "accuracy_std_dev": 0.07649629189542459, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.00888451264222503, "avg_duration_per_task_ms": 348655.1245259166, "avg_output_tokens_per_task": 26392.39317319848, "last_run_timestamp": "2026-10-03T16:13:54.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.522126, "accuracy_std_dev": 0.02474951208754282, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01514130735930736, "avg_duration_per_task_ms": 13491.472582972583, "avg_output_tokens_per_task": 1320.8239538239538, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.513468, "accuracy_std_dev": 0, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.006884491582491581, "avg_duration_per_task_ms": 176603.3367003367, "avg_output_tokens_per_task": 13632.439393939394, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5095547757619738, "accuracy_std_dev": 0.07224117651877167, "run_count": 7, "total_questions": 1378, "avg_cost_per_task_usd": 0.003949532656023222, "avg_duration_per_task_ms": 149830.31567489114, "avg_output_tokens_per_task": 4238.851959361394, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.5056116666666668, "accuracy_std_dev": 0.007570922855827376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012942734006734005, "avg_duration_per_task_ms": 12612.308080808081, "avg_output_tokens_per_task": 1100.3198653198654, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5, "accuracy_std_dev": 0.015120700182200542, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008362104377104377, "avg_duration_per_task_ms": 19874.548821548822, "avg_output_tokens_per_task": 1898.5521885521885, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.49764320000000006, "accuracy_std_dev": 0.01852187333290023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0008495191919191919, "avg_duration_per_task_ms": 13033.880808080808, "avg_output_tokens_per_task": 1851.6353535353535, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.49668788473767883, "accuracy_std_dev": 0.0232226329739954, "run_count": 7, "total_questions": 1258, "avg_cost_per_task_usd": 0.010930109697933227, "avg_duration_per_task_ms": 457519.1629570747, "avg_output_tokens_per_task": 26233.00158982512, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4966332500000001, "accuracy_std_dev": 0.014773079898501197, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008230315656565656, "avg_duration_per_task_ms": 17739.39898989899, "avg_output_tokens_per_task": 1863.909090909091, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.4962063575624083, "accuracy_std_dev": 0.015775167535576604, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.011085070484581497, "avg_duration_per_task_ms": 452241.5058737151, "avg_output_tokens_per_task": 27649.281938325992, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.49588466666666664, "accuracy_std_dev": 0.01555384748400071, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0023523799102132437, "avg_duration_per_task_ms": 10975.11672278339, "avg_output_tokens_per_task": 2330.834455667789, "last_run_timestamp": "2026-10-03T16:58:35.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.4938273950617284, "accuracy_std_dev": 0.04192749999999998, "run_count": 2, "total_questions": 243, "avg_cost_per_task_usd": 0.006835131687242798, "avg_duration_per_task_ms": 360677.2674897119, "avg_output_tokens_per_task": 12627.074074074075, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.48512888888888883, "accuracy_std_dev": 0.005722837707999308, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0023719074074074076, "avg_duration_per_task_ms": 13827.23063973064, "avg_output_tokens_per_task": 2232.787878787879, "last_run_timestamp": "2026-10-03T16:13:49.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.48292587155963307, "accuracy_std_dev": 0.12738947709449225, "run_count": 7, "total_questions": 1308, "avg_cost_per_task_usd": 0.0038326207951070333, "avg_duration_per_task_ms": 105139.82721712538, "avg_output_tokens_per_task": 3476.9533639143733, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.48291236785162295, "accuracy_std_dev": 0.019256962836594973, "run_count": 10, "total_questions": 1941, "avg_cost_per_task_usd": 0.001084483771251932, "avg_duration_per_task_ms": 59779.49407521896, "avg_output_tokens_per_task": 2180.590932509016, "last_run_timestamp": "2026-10-03T16:56:42.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.481818, "accuracy_std_dev": 0.3073381612185509, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.019882339393939394, "avg_duration_per_task_ms": 252178.03333333333, "avg_output_tokens_per_task": 21432.7, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.47763705063291145, "accuracy_std_dev": 0.016207885580790598, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.0012874082278481011, "avg_duration_per_task_ms": 94391.31139240507, "avg_output_tokens_per_task": 2946.448101265823, "last_run_timestamp": "2026-10-03T17:16:32.671Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.4772725, "accuracy_std_dev": 0.0074500120973056095, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002467788720538721, "avg_duration_per_task_ms": 42647.26262626263, "avg_output_tokens_per_task": 2666.861952861953, "last_run_timestamp": "2026-10-03T16:39:56.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.47643079999999993, "accuracy_std_dev": 0.009097059028059564, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.048748201010101015, "avg_duration_per_task_ms": 46893.51111111111, "avg_output_tokens_per_task": 17089.750505050506, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.4758696666666667, "accuracy_std_dev": 0.012294713037182558, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0031561644219977557, "avg_duration_per_task_ms": 51856.26262626263, "avg_output_tokens_per_task": 2228.726150392817, "last_run_timestamp": "2026-10-03T17:41:38.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4750470884396716, "accuracy_std_dev": 0.012160791117742913, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.0013148066961465572, "avg_duration_per_task_ms": 47431.649399873655, "avg_output_tokens_per_task": 2358.5874921036007, "last_run_timestamp": "2026-10-03T16:14:03.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.473274125, "accuracy_std_dev": 0.010392731840540058, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0014396685606060608, "avg_duration_per_task_ms": 66288.69570707071, "avg_output_tokens_per_task": 2272.647095959596, "last_run_timestamp": "2026-10-03T16:57:23.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47210185714285713, "accuracy_std_dev": 0.012910961261419602, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0010585887445887445, "avg_duration_per_task_ms": 186335.3593073593, "avg_output_tokens_per_task": 2390.782828282828, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.46742483824768327, "accuracy_std_dev": 0.006948085005956674, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.002137747262005055, "avg_duration_per_task_ms": 34994.02274641955, "avg_output_tokens_per_task": 2587.217354675653, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.46632999999999997, "accuracy_std_dev": 0.003366999999999981, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023583939393939392, "avg_duration_per_task_ms": 36706.02525252525, "avg_output_tokens_per_task": 2516.760101010101, "last_run_timestamp": "2026-07-18T16:13:31.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46632975000000004, "accuracy_std_dev": 0.020618712591660506, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0007949242424242423, "avg_duration_per_task_ms": 165495.66098484848, "avg_output_tokens_per_task": 2233.563762626263, "last_run_timestamp": "2026-10-03T16:57:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4320986666666667, "accuracy_std_dev": 0.00781631474687543, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010060690235690235, "avg_duration_per_task_ms": 20918.510101010103, "avg_output_tokens_per_task": 1483.5286195286196, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4315373333333334, "accuracy_std_dev": 0.010588221658469791, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010723358585858589, "avg_duration_per_task_ms": 21765.176767676767, "avg_output_tokens_per_task": 1505.858585858586, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4305555, "accuracy_std_dev": 0.007469952225416168, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010262436868686867, "avg_duration_per_task_ms": 17602.7702020202, "avg_output_tokens_per_task": 1517.9292929292928, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.39305708286674135, "accuracy_std_dev": 0.0830628121927015, "run_count": 5, "total_questions": 893, "avg_cost_per_task_usd": 0.015135358342665176, "avg_duration_per_task_ms": 3112776.771556551, "avg_output_tokens_per_task": 15727.527435610302, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.38720499999999997, "accuracy_std_dev": 0.010647388881786938, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00015740101010101008, "avg_duration_per_task_ms": 6591.242424242424, "avg_output_tokens_per_task": 937.0363636363636, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.382155, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.05791383333333333, "avg_duration_per_task_ms": 1031309.4747474748, "avg_output_tokens_per_task": 22994.131313131315, "last_run_timestamp": "2026-10-03T19:39:23.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.3422008461538461, "accuracy_std_dev": 0.08129102269931164, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.00332221217948718, "avg_duration_per_task_ms": 35770.848076923074, "avg_output_tokens_per_task": 1391.8583333333333, "last_run_timestamp": "2026-10-03T16:11:17.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Nebius", "accuracy": 0.338384, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06806519191919191, "avg_duration_per_task_ms": 227007.78787878787, "avg_output_tokens_per_task": 16827.81313131313, "last_run_timestamp": "2026-07-11T08:07:17.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.333333, "accuracy_std_dev": 0.01683499999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00014776262626262624, "avg_duration_per_task_ms": 9430.126262626263, "avg_output_tokens_per_task": 516.290404040404, "last_run_timestamp": "2026-07-18T09:09:45.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.3299665, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00011648232323232323, "avg_duration_per_task_ms": 12253.176767676769, "avg_output_tokens_per_task": 504.4191919191919, "last_run_timestamp": "2026-07-18T09:08:52.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Novita", "accuracy": 0.32468100546448087, "accuracy_std_dev": 0.019489000000000006, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.00011272131147540984, "avg_duration_per_task_ms": 50027.79234972678, "avg_output_tokens_per_task": 481.05464480874315, "last_run_timestamp": "2026-07-18T08:00:15.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DeepInfra", "accuracy": 0.310606, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000030025252525252528, "avg_duration_per_task_ms": 13752.287878787878, "avg_output_tokens_per_task": 429.8787878787879, "last_run_timestamp": "2026-07-18T08:00:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2895623333333333, "accuracy_std_dev": 0.00363692996297092, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006115319865319864, "avg_duration_per_task_ms": 9548.159932659933, "avg_output_tokens_per_task": 578.2474747474747, "last_run_timestamp": "2026-10-01T16:00:58.332Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.28577437499999997, "accuracy_std_dev": 0.01740932382329581, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0014133573232323234, "avg_duration_per_task_ms": 87435.43244949495, "avg_output_tokens_per_task": 18401.633838383837, "last_run_timestamp": "2026-10-03T17:14:12.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.2765752857142857, "accuracy_std_dev": 0.06033612304585108, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00880667316017316, "avg_duration_per_task_ms": 555602.5541125541, "avg_output_tokens_per_task": 21788.924242424244, "last_run_timestamp": "2026-10-03T17:50:31.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.27641208165057063, "accuracy_std_dev": 0.013856526257206988, "run_count": 6, "total_questions": 1139, "avg_cost_per_task_usd": 0.007524540825285337, "avg_duration_per_task_ms": 300907.412642669, "avg_output_tokens_per_task": 33503.4363476734, "last_run_timestamp": "2026-10-03T17:43:23.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DekaLLM", "accuracy": 0.275862, "accuracy_std_dev": 0.01589199999999999, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.000023160919540229885, "avg_duration_per_task_ms": 13795.669540229885, "avg_output_tokens_per_task": 286.0431034482759, "last_run_timestamp": "2026-07-18T08:04:03.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.27409525, "accuracy_std_dev": 0.01585954213360209, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0020373964646464644, "avg_duration_per_task_ms": 44270.92613636364, "avg_output_tokens_per_task": 25522.758838383837, "last_run_timestamp": "2026-10-03T16:55:38.344Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2739361560283688, "accuracy_std_dev": 0.008153796102974918, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.00006610283687943263, "avg_duration_per_task_ms": 8879.929078014184, "avg_output_tokens_per_task": 551.2074468085107, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.273148, "accuracy_std_dev": 0.006113657170630353, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006268181818181819, "avg_duration_per_task_ms": 9553.181818181818, "avg_output_tokens_per_task": 586.439393939394, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.27053954314720813, "accuracy_std_dev": 0.018384550162205943, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.0003133981951494642, "avg_duration_per_task_ms": 375042.1866892273, "avg_output_tokens_per_task": 7590.988155668359, "last_run_timestamp": "2026-10-03T16:18:32.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.2700681612244898, "accuracy_std_dev": 0.016752437809465226, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.00005762346938775511, "avg_duration_per_task_ms": 9734.870408163266, "avg_output_tokens_per_task": 576.8010204081633, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.25757599999999997, "accuracy_std_dev": 0.003366999999999981, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001321409090909091, "avg_duration_per_task_ms": 39858.34595959596, "avg_output_tokens_per_task": 5199.055555555556, "last_run_timestamp": "2026-07-18T08:26:34.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.2504677777777778, "accuracy_std_dev": 0.01889361411093281, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00039258585858585853, "avg_duration_per_task_ms": 66188.20258136925, "avg_output_tokens_per_task": 7528.086419753086, "last_run_timestamp": "2026-10-03T16:12:07.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.23833585714285715, "accuracy_std_dev": 0.09800656715959478, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.001642277056277056, "avg_duration_per_task_ms": 571156.3896103896, "avg_output_tokens_per_task": 5738.018037518037, "last_run_timestamp": "2026-08-29T17:16:14.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.18302071428571431, "accuracy_std_dev": 0.10704161330157577, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00148023088023088, "avg_duration_per_task_ms": 47434.371572871576, "avg_output_tokens_per_task": 1278.541847041847, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.1780305, "accuracy_std_dev": 0.018939499999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.019786199494949495, "avg_duration_per_task_ms": 1989363.3636363635, "avg_output_tokens_per_task": 109668.40909090909, "last_run_timestamp": "2026-10-01T10:42:24.053Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.060606, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.000040161616161616165, "avg_duration_per_task_ms": 3671.1060606060605, "avg_output_tokens_per_task": 93.72222222222223, "last_run_timestamp": "2026-08-01T08:05:34.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.002245, "accuracy_std_dev": 0.0007933738084913065, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003512888888888889, "avg_duration_per_task_ms": 175891.9090909091, "avg_output_tokens_per_task": 3312.4141414141413, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034544065656565656, "avg_duration_per_task_ms": 164367.952020202, "avg_output_tokens_per_task": 3253.931818181818, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.822917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.0087319375, "avg_duration_per_task_ms": 31283.875, "avg_output_tokens_per_task": 9903.5625, "last_run_timestamp": "2026-09-26T08:05:12.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03810672, "avg_duration_per_task_ms": 107015.82, "avg_output_tokens_per_task": 17830.82, "last_run_timestamp": "2026-10-03T08:11:26.752Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8051851111111111, "accuracy_std_dev": 0.020068562864143984, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.87284366, "avg_duration_per_task_ms": 135007.57555555555, "avg_output_tokens_per_task": 5778.728888888889, "last_run_timestamp": "2026-10-03T08:06:43.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8, "accuracy_std_dev": 0.006667000000000034, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.053672830000000005, "avg_duration_per_task_ms": 853001.35, "avg_output_tokens_per_task": 37564.6, "last_run_timestamp": "2026-10-03T08:15:57.209Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.7985185555555555, "accuracy_std_dev": 0.0278934726458882, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.9265346288888889, "avg_duration_per_task_ms": 128415.08888888889, "avg_output_tokens_per_task": 5765.7555555555555, "last_run_timestamp": "2026-10-03T08:17:58.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7952381428571429, "accuracy_std_dev": 0.02038185356374333, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.8750597885714285, "avg_duration_per_task_ms": 147569.17142857143, "avg_output_tokens_per_task": 5729.828571428571, "last_run_timestamp": "2026-10-03T08:21:31.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.020424579999999998, "avg_duration_per_task_ms": 413179.4, "avg_output_tokens_per_task": 22096.3, "last_run_timestamp": "2026-10-03T08:38:04.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7925926666666666, "accuracy_std_dev": 0.01616105838118283, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.8220114666666667, "avg_duration_per_task_ms": 137835.87333333332, "avg_output_tokens_per_task": 5751.835555555555, "last_run_timestamp": "2026-10-03T08:17:58.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.7904040101010101, "accuracy_std_dev": 0.014478087690972844, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.07222263636363636, "avg_duration_per_task_ms": 476697.67171717173, "avg_output_tokens_per_task": 9418.479797979799, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7896079411764706, "accuracy_std_dev": 0.020702777175085597, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.9446278929411763, "avg_duration_per_task_ms": 97552.96117647059, "avg_output_tokens_per_task": 5626.15294117647, "last_run_timestamp": "2026-10-03T08:25:13.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7883334999999999, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09102503, "avg_duration_per_task_ms": 1512645.25, "avg_output_tokens_per_task": 43012.63, "last_run_timestamp": "2026-10-03T08:06:43.541Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.786667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14780512, "avg_duration_per_task_ms": 461958.93, "avg_output_tokens_per_task": 11472.92, "last_run_timestamp": "2026-07-18T08:04:04.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7862320000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.029582717391304348, "avg_duration_per_task_ms": 2435445.304347826, "avg_output_tokens_per_task": 108721.58695652174, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7859532642140468, "accuracy_std_dev": 0.022060806735148097, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028781180602006688, "avg_duration_per_task_ms": 346700.6989966555, "avg_output_tokens_per_task": 11874.802675585284, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7859258888888889, "accuracy_std_dev": 0.023611001599746027, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4867972777777778, "avg_duration_per_task_ms": 127433.78, "avg_output_tokens_per_task": 7132.633333333333, "last_run_timestamp": "2026-10-03T08:09:09.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7857143728222997, "accuracy_std_dev": 0.022636034639976627, "run_count": 6, "total_questions": 287, "avg_cost_per_task_usd": 0.11757857142857142, "avg_duration_per_task_ms": 73705.68641114983, "avg_output_tokens_per_task": 8241.627177700348, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.785, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08538871499999999, "avg_duration_per_task_ms": 41332.83, "avg_output_tokens_per_task": 4686.75, "last_run_timestamp": "2026-10-03T08:21:29.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7847225, "accuracy_std_dev": 0.020833500000000005, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.11493318749999999, "avg_duration_per_task_ms": 147359.25, "avg_output_tokens_per_task": 14387.041666666666, "last_run_timestamp": "2026-10-01T16:00:58.391Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.784471297188755, "accuracy_std_dev": 0.02769278737577711, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.09832240562248996, "avg_duration_per_task_ms": 406639.9036144578, "avg_output_tokens_per_task": 17335.586345381525, "last_run_timestamp": "2026-09-26T08:23:09.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7835159358372455, "accuracy_std_dev": 0.020461293571987414, "run_count": 13, "total_questions": 639, "avg_cost_per_task_usd": 0.08511225195618151, "avg_duration_per_task_ms": 86820.21752738654, "avg_output_tokens_per_task": 4360.830985915493, "last_run_timestamp": "2026-10-03T08:39:04.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7833639159049359, "accuracy_std_dev": 0.011285272864949418, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.10846238025594149, "avg_duration_per_task_ms": 142469.36563071297, "avg_output_tokens_per_task": 11291.52285191956, "last_run_timestamp": "2026-10-03T08:33:36.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02978465, "avg_duration_per_task_ms": 411185.87, "avg_output_tokens_per_task": 14644.16, "last_run_timestamp": "2026-07-18T08:03:40.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06834454, "avg_duration_per_task_ms": 91367.71, "avg_output_tokens_per_task": 11815.81, "last_run_timestamp": "2026-07-18T08:04:03.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.7813334, "accuracy_std_dev": 0.022861647653657863, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.016082664, "avg_duration_per_task_ms": 178213.516, "avg_output_tokens_per_task": 11864.648, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7807408333333333, "accuracy_std_dev": 0.022758947352268594, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.47270033222222213, "avg_duration_per_task_ms": 109166.16777777778, "avg_output_tokens_per_task": 7230.745555555555, "last_run_timestamp": "2026-10-03T08:13:54.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7800003333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2757919799999997, "avg_duration_per_task_ms": 102078.39333333333, "avg_output_tokens_per_task": 5685.653333333334, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7800001249999999, "accuracy_std_dev": 0.02538593223833576, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.18096821249999998, "avg_duration_per_task_ms": 702726.9025, "avg_output_tokens_per_task": 29032.6575, "last_run_timestamp": "2026-10-03T16:06:04.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.78, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12438624000000001, "avg_duration_per_task_ms": 511620.62, "avg_output_tokens_per_task": 12374.38, "last_run_timestamp": "2026-10-03T13:23:27.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01813584, "avg_duration_per_task_ms": 915938.12, "avg_output_tokens_per_task": 36481, "last_run_timestamp": "2026-10-03T08:17:09.266Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12014696000000001, "avg_duration_per_task_ms": 363719.42, "avg_output_tokens_per_task": 9566.28, "last_run_timestamp": "2026-07-18T16:00:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7790475714285714, "accuracy_std_dev": 0.02348347998583042, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4780597828571429, "avg_duration_per_task_ms": 128145.17428571428, "avg_output_tokens_per_task": 7892.805714285714, "last_run_timestamp": "2026-10-03T08:15:57.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7786668, "accuracy_std_dev": 0.02508661263223875, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.095489364, "avg_duration_per_task_ms": 83534.308, "avg_output_tokens_per_task": 4737.02, "last_run_timestamp": "2026-10-03T08:24:07.566Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.778333375, "accuracy_std_dev": 0.020749956493553795, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1860782475, "avg_duration_per_task_ms": 191849.3575, "avg_output_tokens_per_task": 9582.0075, "last_run_timestamp": "2026-10-03T16:10:17.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.77833325, "accuracy_std_dev": 0.011902275693643617, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.19232103999999997, "avg_duration_per_task_ms": 75116.48, "avg_output_tokens_per_task": 6244.94, "last_run_timestamp": "2026-10-03T08:17:58.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.777857107142857, "accuracy_std_dev": 0.024498453027457068, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.5571781464285714, "avg_duration_per_task_ms": 116452.10642857142, "avg_output_tokens_per_task": 7231.447142857143, "last_run_timestamp": "2026-10-03T08:36:29.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7777776666666667, "accuracy_std_dev": 0.021998866415239553, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08069113666666665, "avg_duration_per_task_ms": 131032.13666666667, "avg_output_tokens_per_task": 4578.13, "last_run_timestamp": "2026-10-03T08:38:05.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7772839629629629, "accuracy_std_dev": 0.02231524453767886, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.4874687607407408, "avg_duration_per_task_ms": 125767.80296296296, "avg_output_tokens_per_task": 7179.893333333333, "last_run_timestamp": "2026-10-03T08:32:03.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7770371111111111, "accuracy_std_dev": 0.02456751957110331, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06889805555555557, "avg_duration_per_task_ms": 301183.13333333336, "avg_output_tokens_per_task": 14999.031111111111, "last_run_timestamp": "2026-10-03T16:18:22.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.776803125730994, "accuracy_std_dev": 0.042092438184140796, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.1097883888888889, "avg_duration_per_task_ms": 2869054.269005848, "avg_output_tokens_per_task": 25251.207602339182, "last_run_timestamp": "2026-10-03T16:10:49.371Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7766669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.18507052999999998, "avg_duration_per_task_ms": 70446.4, "avg_output_tokens_per_task": 6214.06, "last_run_timestamp": "2026-10-03T08:13:12.599Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.7766667, "accuracy_std_dev": 0.02955231519542927, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.043956857999999994, "avg_duration_per_task_ms": 213316.834, "avg_output_tokens_per_task": 9541.228, "last_run_timestamp": "2026-10-03T16:15:21.561Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.011385435794128478, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.18964581, "avg_duration_per_task_ms": 65753.47666666667, "avg_output_tokens_per_task": 6306.6466666666665, "last_run_timestamp": "2026-10-03T08:23:24.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.026874233841275463, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15658131666666666, "avg_duration_per_task_ms": 581092.8233333334, "avg_output_tokens_per_task": 12459.696666666667, "last_run_timestamp": "2026-10-03T08:17:09.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7760001, "accuracy_std_dev": 0.021333425001391598, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.44147494200000004, "avg_duration_per_task_ms": 131608.992, "avg_output_tokens_per_task": 7283.092, "last_run_timestamp": "2026-10-03T08:36:29.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.776, "accuracy_std_dev": 0.012364752775530953, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.074485596, "avg_duration_per_task_ms": 47627.076, "avg_output_tokens_per_task": 4707.808, "last_run_timestamp": "2026-10-03T08:37:28.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7755376129032258, "accuracy_std_dev": 0.01731650665001458, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.1579919798387097, "avg_duration_per_task_ms": 333040.58467741933, "avg_output_tokens_per_task": 20363.45564516129, "last_run_timestamp": "2026-10-03T08:32:03.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.775, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029526945, "avg_duration_per_task_ms": 114391.885, "avg_output_tokens_per_task": 11970.57, "last_run_timestamp": "2026-10-03T16:15:04.036Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7748298673469387, "accuracy_std_dev": 0.0364724490240236, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.08125569795918368, "avg_duration_per_task_ms": 1108829.7081632654, "avg_output_tokens_per_task": 26616.873469387756, "last_run_timestamp": "2026-10-03T16:05:34.190Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.7748147777777777, "accuracy_std_dev": 0.029275825597155435, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.023741562222222224, "avg_duration_per_task_ms": 304660.43555555557, "avg_output_tokens_per_task": 11954.117777777778, "last_run_timestamp": "2026-10-03T16:14:24.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7744441666666667, "accuracy_std_dev": 0.015112691911289488, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09511451000000001, "avg_duration_per_task_ms": 133903.96, "avg_output_tokens_per_task": 4573.283333333334, "last_run_timestamp": "2026-10-03T08:23:53.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.7744105858585859, "accuracy_std_dev": 0.019115499999999952, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.24473854545454546, "avg_duration_per_task_ms": 51240.79797979798, "avg_output_tokens_per_task": 4631.454545454545, "last_run_timestamp": "2026-10-03T08:37:57.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.017612503291811795, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12337239714285714, "avg_duration_per_task_ms": 82299.41428571429, "avg_output_tokens_per_task": 4508.834285714286, "last_run_timestamp": "2026-10-03T08:19:27.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.02914423052490644, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4720031333333333, "avg_duration_per_task_ms": 115130.24444444444, "avg_output_tokens_per_task": 7206.9, "last_run_timestamp": "2026-10-03T08:16:37.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.77333325, "accuracy_std_dev": 0.017638436566794497, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4344371716666668, "avg_duration_per_task_ms": 95528.38, "avg_output_tokens_per_task": 5662.913333333333, "last_run_timestamp": "2026-10-03T08:06:28.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.7733332, "accuracy_std_dev": 0.004216264811417815, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007032796, "avg_duration_per_task_ms": 161320.244, "avg_output_tokens_per_task": 16830.132, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02012318, "avg_duration_per_task_ms": 229027.46, "avg_output_tokens_per_task": 10940.04, "last_run_timestamp": "2026-07-18T08:04:46.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7729411176470589, "accuracy_std_dev": 0.02923599978246472, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.5054271811764706, "avg_duration_per_task_ms": 128787.48470588235, "avg_output_tokens_per_task": 7945.302352941177, "last_run_timestamp": "2026-10-03T08:15:30.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.77277775, "accuracy_std_dev": 0.019333070721456368, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4942197533333333, "avg_duration_per_task_ms": 128626.26333333334, "avg_output_tokens_per_task": 8065.555, "last_run_timestamp": "2026-10-03T08:20:22.745Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7727776666666667, "accuracy_std_dev": 0.03324520758278134, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11460699333333334, "avg_duration_per_task_ms": 172737.43666666668, "avg_output_tokens_per_task": 9388.596666666666, "last_run_timestamp": "2026-10-03T16:16:11.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7726666, "accuracy_std_dev": 0.029544518575194266, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.222121268, "avg_duration_per_task_ms": 606766.992, "avg_output_tokens_per_task": 12995.352, "last_run_timestamp": "2026-10-03T08:11:04.620Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7722221666666667, "accuracy_std_dev": 0.0225805089137857, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.009277203333333336, "avg_duration_per_task_ms": 211225.32333333333, "avg_output_tokens_per_task": 4258.733333333334, "last_run_timestamp": "2026-10-03T08:53:37.283Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7718518888888889, "accuracy_std_dev": 0.028765211996609332, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09609024444444446, "avg_duration_per_task_ms": 233464.92444444445, "avg_output_tokens_per_task": 11786.324444444444, "last_run_timestamp": "2026-10-03T16:04:46.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7718518888888889, "accuracy_std_dev": 0.020313041215733758, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1761100022222222, "avg_duration_per_task_ms": 382781.3977777778, "avg_output_tokens_per_task": 27598.153333333332, "last_run_timestamp": "2026-10-03T16:19:10.407Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7718518333333333, "accuracy_std_dev": 0.03201514259473338, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.43984256000000005, "avg_duration_per_task_ms": 80156.35111111111, "avg_output_tokens_per_task": 5674.794444444445, "last_run_timestamp": "2026-10-03T08:11:53.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.77166675, "accuracy_std_dev": 0.027233420032884263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08857231500000001, "avg_duration_per_task_ms": 275121.275, "avg_output_tokens_per_task": 19864.29, "last_run_timestamp": "2026-10-03T08:38:07.639Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.77166675, "accuracy_std_dev": 0.02020715632362706, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09734501, "avg_duration_per_task_ms": 129281.66, "avg_output_tokens_per_task": 5411.39, "last_run_timestamp": "2026-10-03T08:23:24.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.77166675, "accuracy_std_dev": 0.009860203404975986, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.11018350999999998, "avg_duration_per_task_ms": 71664.89, "avg_output_tokens_per_task": 4808.815, "last_run_timestamp": "2026-10-03T08:21:31.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7711112222222222, "accuracy_std_dev": 0.021773400922418973, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5318819755555556, "avg_duration_per_task_ms": 174829.24, "avg_output_tokens_per_task": 10576.931111111111, "last_run_timestamp": "2026-10-03T08:10:31.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7711111666666667, "accuracy_std_dev": 0.016629684897963515, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06979356333333334, "avg_duration_per_task_ms": 35038.676666666666, "avg_output_tokens_per_task": 4742.786666666667, "last_run_timestamp": "2026-10-03T08:30:16.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.7711111111111111, "accuracy_std_dev": 0.03771242713673055, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11559718000000001, "avg_duration_per_task_ms": 294861.4888888889, "avg_output_tokens_per_task": 12304.771111111111, "last_run_timestamp": "2026-10-03T16:16:31.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.771111, "accuracy_std_dev": 0.04085521560339636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4149127733333333, "avg_duration_per_task_ms": 472514.2266666667, "avg_output_tokens_per_task": 36470.113333333335, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7706668, "accuracy_std_dev": 0.010832125062055002, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.100171232, "avg_duration_per_task_ms": 273464.56, "avg_output_tokens_per_task": 22455.776, "last_run_timestamp": "2026-10-03T08:37:40.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7704762857142857, "accuracy_std_dev": 0.03627188525453795, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07425714857142857, "avg_duration_per_task_ms": 258403.66, "avg_output_tokens_per_task": 11728.06, "last_run_timestamp": "2026-10-03T08:42:46.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7700001666666666, "accuracy_std_dev": 0.02168818589629438, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08111510333333333, "avg_duration_per_task_ms": 263647.49333333335, "avg_output_tokens_per_task": 13148.976666666667, "last_run_timestamp": "2026-10-03T08:16:12.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.18774403, "avg_duration_per_task_ms": 78191.08, "avg_output_tokens_per_task": 6339.04, "last_run_timestamp": "2026-10-03T08:11:23.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.77, "accuracy_std_dev": 0.017950332712793934, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.10653246, "avg_duration_per_task_ms": 360324.605, "avg_output_tokens_per_task": 20634.68, "last_run_timestamp": "2026-10-03T08:06:28.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.77, "accuracy_std_dev": 0.023332999999999993, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.35428131, "avg_duration_per_task_ms": 261243.27, "avg_output_tokens_per_task": 8792.41, "last_run_timestamp": "2026-07-18T08:04:09.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7696296666666667, "accuracy_std_dev": 0.03444242845677408, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04467215555555556, "avg_duration_per_task_ms": 206092.15333333332, "avg_output_tokens_per_task": 9425.135555555555, "last_run_timestamp": "2026-10-03T16:14:17.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7696295555555556, "accuracy_std_dev": 0.019654008621998214, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4308894377777778, "avg_duration_per_task_ms": 100253.29111111112, "avg_output_tokens_per_task": 5601.184444444444, "last_run_timestamp": "2026-10-03T08:11:39.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7695238571428571, "accuracy_std_dev": 0.020113036578502115, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.13556214, "avg_duration_per_task_ms": 309797.7, "avg_output_tokens_per_task": 19536.211428571427, "last_run_timestamp": "2026-10-03T08:11:53.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7695237142857143, "accuracy_std_dev": 0.01326512827695538, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10013912285714285, "avg_duration_per_task_ms": 333739.5742857143, "avg_output_tokens_per_task": 20222.222857142857, "last_run_timestamp": "2026-10-03T08:17:48.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7694445, "accuracy_std_dev": 0.02899006747864978, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.551655145, "avg_duration_per_task_ms": 183576.22666666665, "avg_output_tokens_per_task": 10538.358333333334, "last_run_timestamp": "2026-10-03T08:10:31.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.76916675, "accuracy_std_dev": 0.021327567581360528, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1697008125, "avg_duration_per_task_ms": 726758.755, "avg_output_tokens_per_task": 27638.9, "last_run_timestamp": "2026-10-03T16:14:18.634Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.021998731726069008, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09783254, "avg_duration_per_task_ms": 102292.53333333334, "avg_output_tokens_per_task": 23933.02, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0562409, "avg_duration_per_task_ms": 124558.21333333333, "avg_output_tokens_per_task": 4508.253333333333, "last_run_timestamp": "2026-09-26T08:24:47.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.768707, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.18590030612244896, "avg_duration_per_task_ms": 362695.9183673469, "avg_output_tokens_per_task": 6540.102040816327, "last_run_timestamp": "2026-09-29T21:00:05.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.768421252631579, "accuracy_std_dev": 0.0038080000000000336, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.06421121052631579, "avg_duration_per_task_ms": 69460.44210526315, "avg_output_tokens_per_task": 5883.873684210526, "last_run_timestamp": "2026-07-18T08:06:26.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7683335, "accuracy_std_dev": 0.017873086533948198, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14871092, "avg_duration_per_task_ms": 286909.625, "avg_output_tokens_per_task": 13559.755, "last_run_timestamp": "2026-10-03T16:12:29.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7683335, "accuracy_std_dev": 0.01190259075789804, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15200416500000002, "avg_duration_per_task_ms": 286481.905, "avg_output_tokens_per_task": 19800.085, "last_run_timestamp": "2026-10-03T08:10:31.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7683331250000001, "accuracy_std_dev": 0.02254616560214563, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0784358, "avg_duration_per_task_ms": 226865.8275, "avg_output_tokens_per_task": 9339.8125, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.767778, "accuracy_std_dev": 0.009557997105391226, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.055267166666666666, "avg_duration_per_task_ms": 921928.7333333333, "avg_output_tokens_per_task": 38360.306666666664, "last_run_timestamp": "2026-10-03T08:15:16.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7677777777777778, "accuracy_std_dev": 0.03175926811456523, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.5635850022222223, "avg_duration_per_task_ms": 180702.17222222223, "avg_output_tokens_per_task": 10553.803333333333, "last_run_timestamp": "2026-10-03T08:12:06.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7676188571428572, "accuracy_std_dev": 0.04456902618708767, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.13690867142857144, "avg_duration_per_task_ms": 207208.99714285714, "avg_output_tokens_per_task": 7510.56, "last_run_timestamp": "2026-10-03T08:11:26.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7673375906040268, "accuracy_std_dev": 0.01364674745945795, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.02802279865771812, "avg_duration_per_task_ms": 90972.77852348993, "avg_output_tokens_per_task": 17409.81879194631, "last_run_timestamp": "2026-10-03T08:20:48.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7668161793721974, "accuracy_std_dev": 0.039499788335422885, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.028219390134529145, "avg_duration_per_task_ms": 483397.735426009, "avg_output_tokens_per_task": 13354.013452914798, "last_run_timestamp": "2026-10-03T16:10:32.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025223939999999997, "avg_duration_per_task_ms": 165137.1, "avg_output_tokens_per_task": 14240.82, "last_run_timestamp": "2026-09-15T00:00:05.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01215033, "avg_duration_per_task_ms": 551566.9, "avg_output_tokens_per_task": 23566.32, "last_run_timestamp": "2026-10-03T08:18:05.266Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.014401440259764175, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3460653, "avg_duration_per_task_ms": 170899.55333333334, "avg_output_tokens_per_task": 12611.153333333334, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.020727473271535585, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06962397, "avg_duration_per_task_ms": 57268.18666666667, "avg_output_tokens_per_task": 4623.046666666667, "last_run_timestamp": "2026-10-03T08:43:42.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7666666153846154, "accuracy_std_dev": 0.020085283979679725, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.5291991723076925, "avg_duration_per_task_ms": 121619.19230769231, "avg_output_tokens_per_task": 8086.011538461538, "last_run_timestamp": "2026-10-03T08:16:24.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.7666666, "accuracy_std_dev": 0.009428137559454682, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.081377272, "avg_duration_per_task_ms": 89133.128, "avg_output_tokens_per_task": 4543.572, "last_run_timestamp": "2026-10-03T08:30:16.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7666666, "accuracy_std_dev": 0.018856133692780202, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.061894704, "avg_duration_per_task_ms": 48771.608, "avg_output_tokens_per_task": 4659.024, "last_run_timestamp": "2026-10-03T08:19:53.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7666665714285714, "accuracy_std_dev": 0.01951809905012813, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08535362285714285, "avg_duration_per_task_ms": 334395.88857142854, "avg_output_tokens_per_task": 19919.711428571427, "last_run_timestamp": "2026-10-03T08:40:08.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7661904285714286, "accuracy_std_dev": 0.01872337959311179, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019808859999999998, "avg_duration_per_task_ms": 510174.85142857145, "avg_output_tokens_per_task": 13353.934285714286, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7659258888888889, "accuracy_std_dev": 0.030541586931935746, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16052984222222222, "avg_duration_per_task_ms": 227188.16666666666, "avg_output_tokens_per_task": 9718.513333333334, "last_run_timestamp": "2026-10-03T16:12:07.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.765833375, "accuracy_std_dev": 0.020933096139471957, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0119665625, "avg_duration_per_task_ms": 276764.21, "avg_output_tokens_per_task": 13967.9875, "last_run_timestamp": "2026-10-03T16:20:06.645Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7653333999999999, "accuracy_std_dev": 0.032496087971323535, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030119820000000002, "avg_duration_per_task_ms": 258326.172, "avg_output_tokens_per_task": 18849.548, "last_run_timestamp": "2026-10-03T08:13:52.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.7653059591836735, "accuracy_std_dev": 0.005416499999999991, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.02433298979591837, "avg_duration_per_task_ms": 682809.275510204, "avg_output_tokens_per_task": 13469.224489795919, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7651852222222222, "accuracy_std_dev": 0.0219489393759333, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05072059555555555, "avg_duration_per_task_ms": 180142.03555555554, "avg_output_tokens_per_task": 11751.844444444445, "last_run_timestamp": "2026-10-03T16:16:48.795Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.765, "accuracy_std_dev": 0.036930670783239215, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.032793829999999996, "avg_duration_per_task_ms": 164299.645, "avg_output_tokens_per_task": 15398.3, "last_run_timestamp": "2026-10-03T08:19:00.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.765, "accuracy_std_dev": 0.024664481886915866, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0358974525, "avg_duration_per_task_ms": 421646.1775, "avg_output_tokens_per_task": 13842.23, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.765, "accuracy_std_dev": 0.028819341500110663, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.011177185, "avg_duration_per_task_ms": 267604.5325, "avg_output_tokens_per_task": 6349.9575, "last_run_timestamp": "2026-10-03T08:39:56.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7646153076923077, "accuracy_std_dev": 0.017615963863166925, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.019299193846153844, "avg_duration_per_task_ms": 239187.3676923077, "avg_output_tokens_per_task": 17117.927692307694, "last_run_timestamp": "2026-10-03T08:37:24.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7644446666666667, "accuracy_std_dev": 0.020122943113658996, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4976620088888889, "avg_duration_per_task_ms": 132296.16222222222, "avg_output_tokens_per_task": 8163.442222222222, "last_run_timestamp": "2026-10-03T08:17:09.269Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7644445, "accuracy_std_dev": 0.021999102116147078, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06149954333333334, "avg_duration_per_task_ms": 120941.48333333334, "avg_output_tokens_per_task": 4754.513333333333, "last_run_timestamp": "2026-10-03T08:41:33.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7644445, "accuracy_std_dev": 0.03281078940303024, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08256609000000001, "avg_duration_per_task_ms": 48194.92, "avg_output_tokens_per_task": 4847.44, "last_run_timestamp": "2026-10-03T08:29:04.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7644444444444445, "accuracy_std_dev": 0.025141456592255085, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03985802, "avg_duration_per_task_ms": 244206.85555555555, "avg_output_tokens_per_task": 12186.455555555556, "last_run_timestamp": "2026-10-03T16:07:44.182Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029735113333333334, "avg_duration_per_task_ms": 54092.806666666664, "avg_output_tokens_per_task": 4732.093333333333, "last_run_timestamp": "2026-10-03T08:37:59.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7641026153846153, "accuracy_std_dev": 0.016021001176525294, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.017410869230769233, "avg_duration_per_task_ms": 197270.92, "avg_output_tokens_per_task": 19327.204615384617, "last_run_timestamp": "2026-10-03T08:29:06.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7640449662921348, "accuracy_std_dev": 0.01812749711295652, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.06108097303370787, "avg_duration_per_task_ms": 254446.68764044944, "avg_output_tokens_per_task": 11309.25168539326, "last_run_timestamp": "2026-10-03T08:11:26.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.764, "accuracy_std_dev": 0.02585427538338681, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.050899012, "avg_duration_per_task_ms": 154368.784, "avg_output_tokens_per_task": 4603.632, "last_run_timestamp": "2026-10-03T08:39:30.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.7637362417582418, "accuracy_std_dev": 0.023909098570340986, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.03624780952380952, "avg_duration_per_task_ms": 557719.2857142857, "avg_output_tokens_per_task": 12893.100732600733, "last_run_timestamp": "2026-10-03T16:10:49.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7637214417670682, "accuracy_std_dev": 0.03933839868627089, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.11078959437751006, "avg_duration_per_task_ms": 857713.8875502008, "avg_output_tokens_per_task": 10189.773092369478, "last_run_timestamp": "2026-10-03T16:09:05.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7636985890410959, "accuracy_std_dev": 0.021292961012190532, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.059730445205479446, "avg_duration_per_task_ms": 156883.1780821918, "avg_output_tokens_per_task": 9476.835616438357, "last_run_timestamp": "2026-10-03T16:18:52.059Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.7633335, "accuracy_std_dev": 0.015275361427802623, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.028111930000000004, "avg_duration_per_task_ms": 550867.43, "avg_output_tokens_per_task": 16398.445, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01259681, "avg_duration_per_task_ms": 657803.07, "avg_output_tokens_per_task": 25182.87, "last_run_timestamp": "2026-10-03T08:05:49.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.7633335, "accuracy_std_dev": 0.027284651770730018, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.049680514999999995, "avg_duration_per_task_ms": 101520.115, "avg_output_tokens_per_task": 4912.065, "last_run_timestamp": "2026-10-03T08:32:05.590Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02542324, "avg_duration_per_task_ms": 138965.85, "avg_output_tokens_per_task": 13863.01, "last_run_timestamp": "2026-10-03T16:12:10.916Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10827190999999999, "avg_duration_per_task_ms": 51051.39, "avg_output_tokens_per_task": 12753.93, "last_run_timestamp": "2026-07-18T08:03:41.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06322074000000001, "avg_duration_per_task_ms": 968065, "avg_output_tokens_per_task": 43847.04, "last_run_timestamp": "2026-10-03T08:20:05.772Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.043428839999999996, "avg_duration_per_task_ms": 343116.57, "avg_output_tokens_per_task": 9247.74, "last_run_timestamp": "2026-07-18T16:00:10.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7631872962962963, "accuracy_std_dev": 0.03277405103846979, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.08760998316498315, "avg_duration_per_task_ms": 291835.16835016833, "avg_output_tokens_per_task": 9985.114478114478, "last_run_timestamp": "2026-10-03T16:17:43.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7631579766081871, "accuracy_std_dev": 0.021601269198350176, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.025081707602339184, "avg_duration_per_task_ms": 429534.44444444444, "avg_output_tokens_per_task": 12901.88596491228, "last_run_timestamp": "2026-10-03T16:09:19.685Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7627686612903226, "accuracy_std_dev": 0.02051209766552413, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.02082460887096774, "avg_duration_per_task_ms": 113248.77822580645, "avg_output_tokens_per_task": 14257.657258064517, "last_run_timestamp": "2026-10-03T08:37:24.028Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7626666, "accuracy_std_dev": 0.03227647784749755, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.146569104, "avg_duration_per_task_ms": 373419.58, "avg_output_tokens_per_task": 24831.152, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7626487366071428, "accuracy_std_dev": 0.021906600123881723, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.10720718303571429, "avg_duration_per_task_ms": 355012.6004464286, "avg_output_tokens_per_task": 11827.252232142857, "last_run_timestamp": "2026-10-03T16:15:36.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7625, "accuracy_std_dev": 0.01823840575543818, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08220987750000001, "avg_duration_per_task_ms": 289030.8525, "avg_output_tokens_per_task": 15510.735, "last_run_timestamp": "2026-10-03T16:22:04.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.762381, "accuracy_std_dev": 0.013179314517184024, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12107822285714286, "avg_duration_per_task_ms": 467340.5485714286, "avg_output_tokens_per_task": 21773.494285714285, "last_run_timestamp": "2026-10-03T08:54:39.113Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.011330936570096763, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03153908666666666, "avg_duration_per_task_ms": 312202.32, "avg_output_tokens_per_task": 39820.306666666664, "last_run_timestamp": "2026-10-03T08:09:38.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7621211818181818, "accuracy_std_dev": 0.026372448546363295, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05366102181818182, "avg_duration_per_task_ms": 93039.47818181818, "avg_output_tokens_per_task": 4281.278181818182, "last_run_timestamp": "2026-10-03T08:43:08.437Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.7619048571428572, "accuracy_std_dev": 0.017353196366158268, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01491982857142857, "avg_duration_per_task_ms": 401525.7, "avg_output_tokens_per_task": 15852.922857142858, "last_run_timestamp": "2026-10-03T16:17:41.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.020067932249624297, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.026417691428571432, "avg_duration_per_task_ms": 160543.33428571429, "avg_output_tokens_per_task": 8753.874285714286, "last_run_timestamp": "2026-10-03T16:17:22.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.03611523004698906, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07077762857142857, "avg_duration_per_task_ms": 309481.20857142855, "avg_output_tokens_per_task": 12537.965714285714, "last_run_timestamp": "2026-10-03T08:21:29.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.761666625, "accuracy_std_dev": 0.017559268684497514, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08635762500000001, "avg_duration_per_task_ms": 193062.185, "avg_output_tokens_per_task": 11589.4775, "last_run_timestamp": "2026-10-03T16:12:13.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7614815555555555, "accuracy_std_dev": 0.026063039543516672, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12853624, "avg_duration_per_task_ms": 270459.68, "avg_output_tokens_per_task": 11691.066666666668, "last_run_timestamp": "2026-10-03T16:12:29.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7613333999999999, "accuracy_std_dev": 0.019955635240202208, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.054660096000000005, "avg_duration_per_task_ms": 124144.4, "avg_output_tokens_per_task": 4448.496, "last_run_timestamp": "2026-10-03T08:45:16.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7613333999999999, "accuracy_std_dev": 0.022469788112930676, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06195499599999999, "avg_duration_per_task_ms": 143749.252, "avg_output_tokens_per_task": 6410.492, "last_run_timestamp": "2026-10-03T08:22:27.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7610256923076923, "accuracy_std_dev": 0.021218684472687738, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.589651923076923, "avg_duration_per_task_ms": 120803.91923076923, "avg_output_tokens_per_task": 8142.596923076923, "last_run_timestamp": "2026-10-03T08:38:25.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.0335097115932266, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.43242590285714283, "avg_duration_per_task_ms": 94424.92, "avg_output_tokens_per_task": 5553.762857142857, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.04314159352643342, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07637585555555555, "avg_duration_per_task_ms": 225464.4111111111, "avg_output_tokens_per_task": 9525.68, "last_run_timestamp": "2026-10-03T16:14:20.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.014553267185679581, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.056603166666666656, "avg_duration_per_task_ms": 229343.16444444444, "avg_output_tokens_per_task": 10057.042222222222, "last_run_timestamp": "2026-10-03T16:05:34.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7606059090909091, "accuracy_std_dev": 0.02517882197928376, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.16901538363636365, "avg_duration_per_task_ms": 622317.2672727272, "avg_output_tokens_per_task": 38851.54727272727, "last_run_timestamp": "2026-10-03T08:17:06.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7604938148148149, "accuracy_std_dev": 0.020602075859156715, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.656406777037037, "avg_duration_per_task_ms": 73567.76444444444, "avg_output_tokens_per_task": 5646.465185185185, "last_run_timestamp": "2026-10-03T08:42:35.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7604588825910932, "accuracy_std_dev": 0.019116824324139206, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.16499732388663968, "avg_duration_per_task_ms": 837460.5141700405, "avg_output_tokens_per_task": 31365.983805668016, "last_run_timestamp": "2026-10-03T16:08:33.546Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7600502512562815, "accuracy_std_dev": 0.021538150907169366, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.04400205025125628, "avg_duration_per_task_ms": 537418.8140703518, "avg_output_tokens_per_task": 22676.286432160803, "last_run_timestamp": "2026-10-03T16:15:26.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7600001249999999, "accuracy_std_dev": 0.03349974295213884, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0116075025, "avg_duration_per_task_ms": 155751.2425, "avg_output_tokens_per_task": 14262.745, "last_run_timestamp": "2026-10-03T16:12:33.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7600001111111111, "accuracy_std_dev": 0.02012312716720875, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16286194444444443, "avg_duration_per_task_ms": 435512.70222222223, "avg_output_tokens_per_task": 9533.81111111111, "last_run_timestamp": "2026-10-03T16:16:10.656Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.76, "accuracy_std_dev": 0.02735228340559523, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5190067183333332, "avg_duration_per_task_ms": 221047.60166666665, "avg_output_tokens_per_task": 10200.993333333334, "last_run_timestamp": "2026-10-03T08:20:09.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.76, "accuracy_std_dev": 0.020870148806369355, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.0067541, "avg_duration_per_task_ms": 260092.14, "avg_output_tokens_per_task": 15338.358, "last_run_timestamp": "2026-10-03T08:38:03.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.76, "accuracy_std_dev": 0.013984245106547586, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.16696927200000003, "avg_duration_per_task_ms": 223419.404, "avg_output_tokens_per_task": 11293.668, "last_run_timestamp": "2026-10-03T16:10:46.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.76, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12531742999999998, "avg_duration_per_task_ms": 312784.77, "avg_output_tokens_per_task": 20756.6, "last_run_timestamp": "2026-10-03T08:29:42.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015769889999999998, "avg_duration_per_task_ms": 1152795.83, "avg_output_tokens_per_task": 36264.12, "last_run_timestamp": "2026-10-03T08:16:24.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.76, "accuracy_std_dev": 0.019626323972325197, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02215426, "avg_duration_per_task_ms": 122620.84, "avg_output_tokens_per_task": 8751.653333333334, "last_run_timestamp": "2026-10-03T16:07:50.330Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0333696, "avg_duration_per_task_ms": 112327.2, "avg_output_tokens_per_task": 8636.08, "last_run_timestamp": "2026-07-11T08:05:22.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.30272262, "avg_duration_per_task_ms": 355584.56, "avg_output_tokens_per_task": 10613.82, "last_run_timestamp": "2026-10-03T18:42:35.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06203801500000001, "avg_duration_per_task_ms": 146531.195, "avg_output_tokens_per_task": 14707.395, "last_run_timestamp": "2026-10-03T08:15:00.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": 0.02177351432359967, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03884535333333333, "avg_duration_per_task_ms": 362781.7866666667, "avg_output_tokens_per_task": 12784.353333333333, "last_run_timestamp": "2026-10-03T08:09:52.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07523408, "avg_duration_per_task_ms": 676626.08, "avg_output_tokens_per_task": 16285.62, "last_run_timestamp": "2026-10-03T08:15:42.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.76, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12663513999999998, "avg_duration_per_task_ms": 73292.09, "avg_output_tokens_per_task": 9837.08, "last_run_timestamp": "2026-07-18T08:04:45.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.76, "accuracy_std_dev": 1.1102230246251565e-16, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05899852666666667, "avg_duration_per_task_ms": 72162.04666666666, "avg_output_tokens_per_task": 4660.633333333333, "last_run_timestamp": "2026-10-03T08:13:12.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.020547857418718854, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011870755, "avg_duration_per_task_ms": 198678.775, "avg_output_tokens_per_task": 4474.35, "last_run_timestamp": "2026-10-03T08:39:24.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014824000000000002, "avg_duration_per_task_ms": 335920.77, "avg_output_tokens_per_task": 4448.51, "last_run_timestamp": "2026-09-12T08:11:26.740Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7599998888888889, "accuracy_std_dev": 0.031894974255600735, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08708899555555556, "avg_duration_per_task_ms": 223739.0377777778, "avg_output_tokens_per_task": 9920.264444444445, "last_run_timestamp": "2026-10-03T16:09:05.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7599998333333334, "accuracy_std_dev": 0.01333333333437501, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03796188999999999, "avg_duration_per_task_ms": 83545.53666666667, "avg_output_tokens_per_task": 4558.626666666667, "last_run_timestamp": "2026-10-03T08:35:08.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7595079060402684, "accuracy_std_dev": 0.0216087995861768, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.1067814966442953, "avg_duration_per_task_ms": 41164.906040268455, "avg_output_tokens_per_task": 4181.802013422819, "last_run_timestamp": "2026-10-03T21:42:23.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7594443333333334, "accuracy_std_dev": 0.015265178644731146, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.008844093333333332, "avg_duration_per_task_ms": 361189.67333333334, "avg_output_tokens_per_task": 14258.985, "last_run_timestamp": "2026-10-03T08:39:28.580Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7593333, "accuracy_std_dev": 0.016452043441773433, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.083344808, "avg_duration_per_task_ms": 106115.21, "avg_output_tokens_per_task": 23741.31, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7592592222222222, "accuracy_std_dev": 0.020714267497322178, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.41923416888888887, "avg_duration_per_task_ms": 97992.62444444445, "avg_output_tokens_per_task": 5633.653333333334, "last_run_timestamp": "2026-10-03T08:15:30.343Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.75916675, "accuracy_std_dev": 0.03187424291159712, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.072068195, "avg_duration_per_task_ms": 294630.755, "avg_output_tokens_per_task": 11469.0825, "last_run_timestamp": "2026-10-03T16:21:33.429Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7590477142857143, "accuracy_std_dev": 0.013997181448055748, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.24529245999999996, "avg_duration_per_task_ms": 151053.55142857143, "avg_output_tokens_per_task": 23224.54, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7590477142857143, "accuracy_std_dev": 0.034992667843071215, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08880086285714285, "avg_duration_per_task_ms": 68112.38857142857, "avg_output_tokens_per_task": 12261.222857142857, "last_run_timestamp": "2026-10-03T16:15:11.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7588384343434343, "accuracy_std_dev": 0.024611570108132058, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20684339646464647, "avg_duration_per_task_ms": 493166.0580808081, "avg_output_tokens_per_task": 20296.997474747473, "last_run_timestamp": "2026-10-03T08:19:53.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7586668, "accuracy_std_dev": 0.01654634971708262, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.071547176, "avg_duration_per_task_ms": 127553.92, "avg_output_tokens_per_task": 4587.528, "last_run_timestamp": "2026-10-03T09:01:39.121Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.7585185555555556, "accuracy_std_dev": 0.03232214986356059, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01401343777777778, "avg_duration_per_task_ms": 177576.9088888889, "avg_output_tokens_per_task": 14965.488888888889, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.758333375, "accuracy_std_dev": 0.019649939923174698, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09441886249999999, "avg_duration_per_task_ms": 355424.545, "avg_output_tokens_per_task": 12862.18, "last_run_timestamp": "2026-10-03T16:13:59.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.40697005500000005, "avg_duration_per_task_ms": 402356.405, "avg_output_tokens_per_task": 35595.7, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7583331666666667, "accuracy_std_dev": 0.013844473427047428, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09591996333333334, "avg_duration_per_task_ms": 214673.9, "avg_output_tokens_per_task": 9689.54, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7580951428571429, "accuracy_std_dev": 0.025125616384357164, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5482415914285714, "avg_duration_per_task_ms": 178558.36285714287, "avg_output_tokens_per_task": 10788.502857142857, "last_run_timestamp": "2026-10-03T08:08:31.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.758, "accuracy_std_dev": 0.03709447398198281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.079319272, "avg_duration_per_task_ms": 620750.224, "avg_output_tokens_per_task": 10895.356, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7579964141414142, "accuracy_std_dev": 0.023533015514113357, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20713992929292932, "avg_duration_per_task_ms": 664979.7727272727, "avg_output_tokens_per_task": 53099.67929292929, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7577778333333334, "accuracy_std_dev": 0.024545289459260582, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.026035279999999997, "avg_duration_per_task_ms": 169995.66166666665, "avg_output_tokens_per_task": 18134.881666666668, "last_run_timestamp": "2026-10-03T08:33:35.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.04629468145358481, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12420428666666666, "avg_duration_per_task_ms": 116279.86, "avg_output_tokens_per_task": 19029.046666666665, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.35916456, "avg_duration_per_task_ms": 165675.65333333332, "avg_output_tokens_per_task": 12758.866666666667, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.7576270508474576, "accuracy_std_dev": 0.017540967140446465, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.11473522711864406, "avg_duration_per_task_ms": 198579.0440677966, "avg_output_tokens_per_task": 7218.661016949152, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.757619, "accuracy_std_dev": 0.014553057204587645, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3625878914285714, "avg_duration_per_task_ms": 337462.2485714286, "avg_output_tokens_per_task": 10492.708571428571, "last_run_timestamp": "2026-10-03T08:37:29.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.757437664845173, "accuracy_std_dev": 0.028522387468022964, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.08139836976320584, "avg_duration_per_task_ms": 124217.46448087432, "avg_output_tokens_per_task": 12954.497267759563, "last_run_timestamp": "2026-10-03T16:24:57.626Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7573627951807229, "accuracy_std_dev": 0.03546880209832297, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.1904817610441767, "avg_duration_per_task_ms": 475270.8333333333, "avg_output_tokens_per_task": 28088.13855421687, "last_run_timestamp": "2026-10-03T16:15:58.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.757037, "accuracy_std_dev": 0.028822348820933333, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5493601955555556, "avg_duration_per_task_ms": 192839.5377777778, "avg_output_tokens_per_task": 10676.25111111111, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17767661, "avg_duration_per_task_ms": 84052.21, "avg_output_tokens_per_task": 6226.55, "last_run_timestamp": "2026-10-03T08:17:58.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02875953, "avg_duration_per_task_ms": 155469.3, "avg_output_tokens_per_task": 15776.48, "last_run_timestamp": "2026-10-03T08:38:11.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7566667777777778, "accuracy_std_dev": 0.031583837092826035, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07302445555555555, "avg_duration_per_task_ms": 400612.2711111111, "avg_output_tokens_per_task": 9214.242222222223, "last_run_timestamp": "2026-10-03T16:19:32.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.756525067114094, "accuracy_std_dev": 0.028844409691384666, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.0700060134228188, "avg_duration_per_task_ms": 265527.27293064876, "avg_output_tokens_per_task": 11642.608501118568, "last_run_timestamp": "2026-10-03T13:30:10.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7563528834080718, "accuracy_std_dev": 0.046239678716204095, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.07944193273542602, "avg_duration_per_task_ms": 387708.71524663677, "avg_output_tokens_per_task": 12769.977578475336, "last_run_timestamp": "2026-10-03T16:20:30.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7561905714285714, "accuracy_std_dev": 0.0243929472000011, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1949313942857143, "avg_duration_per_task_ms": 124043.37428571428, "avg_output_tokens_per_task": 7684.874285714286, "last_run_timestamp": "2026-10-03T08:17:09.273Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7561538461538462, "accuracy_std_dev": 0.031169392555582855, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.2008702830769231, "avg_duration_per_task_ms": 120980.52153846154, "avg_output_tokens_per_task": 7797.5623076923075, "last_run_timestamp": "2026-10-03T08:16:37.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.756, "accuracy_std_dev": 0.02048314212224288, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022952103999999997, "avg_duration_per_task_ms": 125535.02, "avg_output_tokens_per_task": 15581.832, "last_run_timestamp": "2026-10-03T08:16:37.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.756, "accuracy_std_dev": 0.021745967699782896, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.006672712000000001, "avg_duration_per_task_ms": 144287.16, "avg_output_tokens_per_task": 4265.224, "last_run_timestamp": "2026-10-03T08:12:06.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7557079178082191, "accuracy_std_dev": 0.01607886717264481, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.2608585205479452, "avg_duration_per_task_ms": 125025.47260273973, "avg_output_tokens_per_task": 7152.054794520548, "last_run_timestamp": "2026-10-03T08:05:09.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7557075582191781, "accuracy_std_dev": 0.014317957738642594, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.03211151712328767, "avg_duration_per_task_ms": 2014288.493150685, "avg_output_tokens_per_task": 47550.393835616436, "last_run_timestamp": "2026-10-03T16:09:32.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7555555555555555, "accuracy_std_dev": 0.02266234163693451, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.34417582888888887, "avg_duration_per_task_ms": 219931.23555555556, "avg_output_tokens_per_task": 11911.208888888888, "last_run_timestamp": "2026-10-03T08:23:27.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7555555, "accuracy_std_dev": 0.03119039105103792, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.0069151533333333326, "avg_duration_per_task_ms": 128985.71333333333, "avg_output_tokens_per_task": 4275.633333333333, "last_run_timestamp": "2026-10-03T08:51:15.407Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.755, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07160828000000001, "avg_duration_per_task_ms": 331234.47, "avg_output_tokens_per_task": 13566.125, "last_run_timestamp": "2026-10-03T08:43:45.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7549019380804953, "accuracy_std_dev": 0.022500075722423546, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.020211260061919506, "avg_duration_per_task_ms": 487189.79102167184, "avg_output_tokens_per_task": 17152.354489164085, "last_run_timestamp": "2026-10-03T08:34:10.987Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7546668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.19784133199999998, "avg_duration_per_task_ms": 328733.652, "avg_output_tokens_per_task": 12530.504, "last_run_timestamp": "2026-10-03T08:11:26.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7546666, "accuracy_std_dev": 0.0308113572151569, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.025647999999999997, "avg_duration_per_task_ms": 294521.988, "avg_output_tokens_per_task": 13284.328, "last_run_timestamp": "2026-10-03T16:12:20.035Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7545098823529411, "accuracy_std_dev": 0.035662554927909464, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.19809168117647058, "avg_duration_per_task_ms": 103146.24235294118, "avg_output_tokens_per_task": 7666.7011764705885, "last_run_timestamp": "2026-10-03T08:20:09.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7545043243243243, "accuracy_std_dev": 0.0263378748741976, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.023569141891891895, "avg_duration_per_task_ms": 316460.4864864865, "avg_output_tokens_per_task": 4132.989864864865, "last_run_timestamp": "2026-10-03T08:34:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.7544445, "accuracy_std_dev": 0.02678215501504689, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08861642333333332, "avg_duration_per_task_ms": 293005.13333333336, "avg_output_tokens_per_task": 12023.27, "last_run_timestamp": "2026-10-03T08:37:04.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7544445, "accuracy_std_dev": 0.025067718163340406, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008447913333333333, "avg_duration_per_task_ms": 189202.61333333334, "avg_output_tokens_per_task": 4437.176666666666, "last_run_timestamp": "2026-10-03T08:32:50.838Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7544443333333334, "accuracy_std_dev": 0.02786659578340269, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008521756666666666, "avg_duration_per_task_ms": 190718.40666666668, "avg_output_tokens_per_task": 4439.933333333333, "last_run_timestamp": "2026-10-03T08:37:24.517Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7542855714285714, "accuracy_std_dev": 0.03175878188495785, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.010189677142857144, "avg_duration_per_task_ms": 226644.26285714286, "avg_output_tokens_per_task": 15464.714285714286, "last_run_timestamp": "2026-10-03T16:05:34.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.75416675, "accuracy_std_dev": 0.02012119874504252, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.052130175, "avg_duration_per_task_ms": 124686.1225, "avg_output_tokens_per_task": 4528.42, "last_run_timestamp": "2026-10-03T13:20:27.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7541132888482633, "accuracy_std_dev": 0.033056453595789424, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.1956822632541134, "avg_duration_per_task_ms": 586732.365630713, "avg_output_tokens_per_task": 27848.411334552104, "last_run_timestamp": "2026-10-03T08:06:43.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.754074, "accuracy_std_dev": 0.039182303332556186, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19633914666666663, "avg_duration_per_task_ms": 104140.85111111111, "avg_output_tokens_per_task": 7619.131111111111, "last_run_timestamp": "2026-10-03T08:06:43.541Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7540738888888889, "accuracy_std_dev": 0.021646890628368387, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07777800888888889, "avg_duration_per_task_ms": 250139.2111111111, "avg_output_tokens_per_task": 9830.866666666667, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.754059358166189, "accuracy_std_dev": 0.029275519612321662, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.16745791404011462, "avg_duration_per_task_ms": 461873.38681948424, "avg_output_tokens_per_task": 23575.730659025787, "last_run_timestamp": "2026-08-29T17:19:03.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.754, "accuracy_std_dev": 0.021385259722528513, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.022165832, "avg_duration_per_task_ms": 128895.106, "avg_output_tokens_per_task": 17780.99, "last_run_timestamp": "2026-10-03T08:11:39.228Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.754, "accuracy_std_dev": 0.02048284919633984, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08895154, "avg_duration_per_task_ms": 301180.94, "avg_output_tokens_per_task": 9633.452, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.753914899328859, "accuracy_std_dev": 0.040895046741628756, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.07723322147651007, "avg_duration_per_task_ms": 229349.62863534674, "avg_output_tokens_per_task": 14123.306487695749, "last_run_timestamp": "2026-10-03T08:50:19.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7538975456570156, "accuracy_std_dev": 0.023333857993043206, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.013686612472160357, "avg_duration_per_task_ms": 214524.49888641425, "avg_output_tokens_per_task": 16219.260579064588, "last_run_timestamp": "2026-10-03T08:34:35.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7538888333333333, "accuracy_std_dev": 0.019140560322838583, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.025353155, "avg_duration_per_task_ms": 294929.6716666667, "avg_output_tokens_per_task": 17480.86, "last_run_timestamp": "2026-10-03T08:34:34.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06322421, "avg_duration_per_task_ms": 228589.91, "avg_output_tokens_per_task": 10641.07, "last_run_timestamp": "2026-10-03T16:12:33.507Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7533335, "accuracy_std_dev": 0.01805358280000584, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15314664666666666, "avg_duration_per_task_ms": 312626.37, "avg_output_tokens_per_task": 20191.006666666668, "last_run_timestamp": "2026-10-03T08:43:11.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7533333999999999, "accuracy_std_dev": 0.015776240928687683, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.138680804, "avg_duration_per_task_ms": 204157.96, "avg_output_tokens_per_task": 8340.072, "last_run_timestamp": "2026-10-03T08:52:57.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.027755626724844277, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5350316133333334, "avg_duration_per_task_ms": 188978.14666666667, "avg_output_tokens_per_task": 10298.875555555556, "last_run_timestamp": "2026-10-03T08:09:23.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02739730544739352, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19939614444444445, "avg_duration_per_task_ms": 129460.44888888889, "avg_output_tokens_per_task": 7850.873333333333, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.027216688781374985, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02417726666666667, "avg_duration_per_task_ms": 450171.92, "avg_output_tokens_per_task": 14030.733333333334, "last_run_timestamp": "2026-10-03T08:13:54.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11451421333333334, "avg_duration_per_task_ms": 146387.56666666668, "avg_output_tokens_per_task": 17546.113333333335, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.29158895333333334, "avg_duration_per_task_ms": 171961.5, "avg_output_tokens_per_task": 10696.2, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06088320666666667, "avg_duration_per_task_ms": 1112758.12, "avg_output_tokens_per_task": 52398.04, "last_run_timestamp": "2026-10-03T16:14:15.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7533333, "accuracy_std_dev": 0.023475765282733617, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.021858692000000002, "avg_duration_per_task_ms": 119951.828, "avg_output_tokens_per_task": 15133.982, "last_run_timestamp": "2026-10-03T08:40:00.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7533332500000001, "accuracy_std_dev": 0.014907082584714572, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.243283165, "avg_duration_per_task_ms": 338011.175, "avg_output_tokens_per_task": 17182.585, "last_run_timestamp": "2026-10-03T08:32:03.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.032659863237109024, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12209446666666665, "avg_duration_per_task_ms": 147223.81333333332, "avg_output_tokens_per_task": 19831.306666666667, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2515843, "avg_duration_per_task_ms": 421006.06, "avg_output_tokens_per_task": 32340.48, "last_run_timestamp": "2026-10-01T08:00:24.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.753333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.25651981, "avg_duration_per_task_ms": 445171.22, "avg_output_tokens_per_task": 25359.93, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0196027, "avg_duration_per_task_ms": 76385.02, "avg_output_tokens_per_task": 9950, "last_run_timestamp": "2026-07-18T08:04:02.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07618375999999999, "avg_duration_per_task_ms": 574082.66, "avg_output_tokens_per_task": 14524.72, "last_run_timestamp": "2026-10-03T08:19:27.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.752923926900585, "accuracy_std_dev": 0.02558177145020674, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.006783909356725145, "avg_duration_per_task_ms": 266624.59064327483, "avg_output_tokens_per_task": 11932.982456140351, "last_run_timestamp": "2026-08-29T16:33:29.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7528571428571429, "accuracy_std_dev": 0.043949114567478, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1576592571428571, "avg_duration_per_task_ms": 349242.77428571426, "avg_output_tokens_per_task": 22490.9, "last_run_timestamp": "2026-10-03T08:08:18.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.03021638651133634, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5157013133333334, "avg_duration_per_task_ms": 178548.1511111111, "avg_output_tokens_per_task": 7818.084444444445, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7523456666666667, "accuracy_std_dev": 0.03581868589830568, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.22564089555555555, "avg_duration_per_task_ms": 98845.40814814816, "avg_output_tokens_per_task": 7557.676296296297, "last_run_timestamp": "2026-10-03T08:43:57.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7522221666666666, "accuracy_std_dev": 0.01511278994115765, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.44058496666666663, "avg_duration_per_task_ms": 458389.25, "avg_output_tokens_per_task": 35681.92333333333, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.752, "accuracy_std_dev": 0.015434478002187193, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036405735999999994, "avg_duration_per_task_ms": 202702.796, "avg_output_tokens_per_task": 18338.232, "last_run_timestamp": "2026-09-26T08:01:18.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7519045714285715, "accuracy_std_dev": 0.03226166192184128, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.016922877142857146, "avg_duration_per_task_ms": 77003.04285714286, "avg_output_tokens_per_task": 4241.571428571428, "last_run_timestamp": "2026-10-03T08:48:31.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.751852, "accuracy_std_dev": 0.03366914875008805, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021374442222222224, "avg_duration_per_task_ms": 173854.87555555557, "avg_output_tokens_per_task": 14192.557777777778, "last_run_timestamp": "2026-10-03T08:32:03.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.7516778389261746, "accuracy_std_dev": 0.008511827118127408, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.046181536912751675, "avg_duration_per_task_ms": 406586.1208053691, "avg_output_tokens_per_task": 7644.885906040268, "last_run_timestamp": "2026-07-25T08:07:26.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7511113333333332, "accuracy_std_dev": 0.014740562999952068, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.14565638333333333, "avg_duration_per_task_ms": 139837.15, "avg_output_tokens_per_task": 20029.903333333332, "last_run_timestamp": "2026-10-03T08:32:03.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7511113333333332, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48754586, "avg_duration_per_task_ms": 141651.29333333333, "avg_output_tokens_per_task": 7913.286666666667, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.751111, "accuracy_std_dev": 0.023726809342082964, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014194133333333334, "avg_duration_per_task_ms": 218507.32444444444, "avg_output_tokens_per_task": 15286.804444444444, "last_run_timestamp": "2026-10-03T16:07:05.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7510256923076923, "accuracy_std_dev": 0.024103851303776277, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.3703844146153846, "avg_duration_per_task_ms": 173913.4323076923, "avg_output_tokens_per_task": 11840.180769230768, "last_run_timestamp": "2026-10-03T08:17:09.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7510038393574298, "accuracy_std_dev": 0.038821325135033696, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.13673703614457833, "avg_duration_per_task_ms": 427247.5200803213, "avg_output_tokens_per_task": 19689.73092369478, "last_run_timestamp": "2026-10-03T16:14:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7508331250000001, "accuracy_std_dev": 0.02331842529759192, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.115564, "avg_duration_per_task_ms": 319983.2525, "avg_output_tokens_per_task": 9741.9925, "last_run_timestamp": "2026-10-03T16:14:01.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7506265313283209, "accuracy_std_dev": 0.029800800965536136, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.06770081704260651, "avg_duration_per_task_ms": 350619.045112782, "avg_output_tokens_per_task": 10643.185463659147, "last_run_timestamp": "2026-08-29T17:16:16.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7505960686695279, "accuracy_std_dev": 0.014417471897786718, "run_count": 14, "total_questions": 699, "avg_cost_per_task_usd": 0.05111362374821173, "avg_duration_per_task_ms": 282613.5150214592, "avg_output_tokens_per_task": 30005.542203147354, "last_run_timestamp": "2026-10-03T16:08:50.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.75, "accuracy_std_dev": 0.04384306609373936, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.50077502, "avg_duration_per_task_ms": 139265.845, "avg_output_tokens_per_task": 7986.47, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.75, "accuracy_std_dev": 0.03766314634582015, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04774656, "avg_duration_per_task_ms": 476512.7733333333, "avg_output_tokens_per_task": 9148.09, "last_run_timestamp": "2026-08-29T16:10:33.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7495237142857143, "accuracy_std_dev": 0.027338579222746164, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11086225999999999, "avg_duration_per_task_ms": 273444.3714285714, "avg_output_tokens_per_task": 12185.84, "last_run_timestamp": "2026-10-03T08:09:23.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7493332, "accuracy_std_dev": 0.011623875814890659, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.177717976, "avg_duration_per_task_ms": 166159.384, "avg_output_tokens_per_task": 12813.752, "last_run_timestamp": "2026-10-03T08:12:20.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.749228462962963, "accuracy_std_dev": 0.019134704306557976, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.006892067901234569, "avg_duration_per_task_ms": 186483.29166666666, "avg_output_tokens_per_task": 12568.354938271605, "last_run_timestamp": "2026-10-03T08:37:24.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.749166625, "accuracy_std_dev": 0.02787621821076839, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021932149999999997, "avg_duration_per_task_ms": 451561.2275, "avg_output_tokens_per_task": 15251.1375, "last_run_timestamp": "2026-10-03T16:20:50.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7490367167630058, "accuracy_std_dev": 0.02179500605877504, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.011687421965317918, "avg_duration_per_task_ms": 205541.59826589594, "avg_output_tokens_per_task": 4094.841040462428, "last_run_timestamp": "2026-10-03T08:38:03.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.748889, "accuracy_std_dev": 0.019626097517789383, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.24296280222222222, "avg_duration_per_task_ms": 310167.38, "avg_output_tokens_per_task": 9997.322222222223, "last_run_timestamp": "2026-10-03T08:39:17.397Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7488888888888889, "accuracy_std_dev": 0.050965958138620863, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07998689333333331, "avg_duration_per_task_ms": 240287.86666666667, "avg_output_tokens_per_task": 9722.404444444444, "last_run_timestamp": "2026-10-03T16:12:22.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7488737702702704, "accuracy_std_dev": 0.035681240826082414, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.3075426081081082, "avg_duration_per_task_ms": 149494.7972972973, "avg_output_tokens_per_task": 20582.41554054054, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7485381177944861, "accuracy_std_dev": 0.021900710861750612, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.028614501253132835, "avg_duration_per_task_ms": 631491.8922305765, "avg_output_tokens_per_task": 22158.218045112782, "last_run_timestamp": "2026-10-03T16:12:26.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7483332500000001, "accuracy_std_dev": 0.02803765820916874, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.338020715, "avg_duration_per_task_ms": 209359.69166666668, "avg_output_tokens_per_task": 11805.883333333333, "last_run_timestamp": "2026-10-03T08:02:27.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7481482222222222, "accuracy_std_dev": 0.03186030551146598, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5489024666666666, "avg_duration_per_task_ms": 183789.9622222222, "avg_output_tokens_per_task": 10558.302222222223, "last_run_timestamp": "2026-10-03T08:38:04.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7478787272727273, "accuracy_std_dev": 0.021801232475965104, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03953649272727272, "avg_duration_per_task_ms": 299791.2454545455, "avg_output_tokens_per_task": 24986.36, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.0267822656461116, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07834820999999999, "avg_duration_per_task_ms": 157847.65, "avg_output_tokens_per_task": 8613.366666666667, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7477776666666667, "accuracy_std_dev": 0.011166340517625106, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17740497333333335, "avg_duration_per_task_ms": 310994.1566666667, "avg_output_tokens_per_task": 10783.98, "last_run_timestamp": "2026-10-03T08:17:48.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7477776666666667, "accuracy_std_dev": 0.016063016338436865, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07122743666666666, "avg_duration_per_task_ms": 145473.51333333334, "avg_output_tokens_per_task": 4717.823333333334, "last_run_timestamp": "2026-10-03T08:39:53.772Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7477678928571428, "accuracy_std_dev": 0.025597104175528147, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.0058840625, "avg_duration_per_task_ms": 283531.1651785714, "avg_output_tokens_per_task": 12883.198660714286, "last_run_timestamp": "2026-10-03T16:13:42.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7474747407407407, "accuracy_std_dev": 0.061334593171164214, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.07039795622895623, "avg_duration_per_task_ms": 90683.95286195286, "avg_output_tokens_per_task": 12893.693602693602, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7474074444444444, "accuracy_std_dev": 0.025809327224909778, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4897610111111111, "avg_duration_per_task_ms": 194827.99555555556, "avg_output_tokens_per_task": 10055.273333333333, "last_run_timestamp": "2026-10-03T08:17:06.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.74722225, "accuracy_std_dev": 0.048339794907379365, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08423352666666667, "avg_duration_per_task_ms": 189496.69333333333, "avg_output_tokens_per_task": 8052.658333333334, "last_run_timestamp": "2026-10-03T16:25:24.079Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7470370000000001, "accuracy_std_dev": 0.028671978645437867, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.5621734177777777, "avg_duration_per_task_ms": 202509.61777777778, "avg_output_tokens_per_task": 9856.906666666666, "last_run_timestamp": "2026-10-03T08:21:15.324Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.7469343143812709, "accuracy_std_dev": 0.029603801347968823, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06045377926421405, "avg_duration_per_task_ms": 1570213.8294314381, "avg_output_tokens_per_task": 8606.695652173914, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1787217, "avg_duration_per_task_ms": 84226.38, "avg_output_tokens_per_task": 6412.78, "last_run_timestamp": "2026-10-03T08:15:16.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02698528, "avg_duration_per_task_ms": 147907.84, "avg_output_tokens_per_task": 11065.38, "last_run_timestamp": "2026-07-18T08:04:04.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.027034372269390698, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015178880000000002, "avg_duration_per_task_ms": 145373.45333333334, "avg_output_tokens_per_task": 16435.604444444445, "last_run_timestamp": "2026-10-03T08:25:27.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.023726809342082992, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21412848666666662, "avg_duration_per_task_ms": 142371.99333333335, "avg_output_tokens_per_task": 22756.126666666667, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18399340666666666, "avg_duration_per_task_ms": 116870.40666666666, "avg_output_tokens_per_task": 5117.2266666666665, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.746666625, "accuracy_std_dev": 0.05695993231197147, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.019652835, "avg_duration_per_task_ms": 219535.91, "avg_output_tokens_per_task": 12359.6225, "last_run_timestamp": "2026-10-03T16:19:59.478Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.746666625, "accuracy_std_dev": 0.03349969319985448, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.087697375, "avg_duration_per_task_ms": 258190.44, "avg_output_tokens_per_task": 12176.68, "last_run_timestamp": "2026-10-03T08:20:48.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665555555556, "accuracy_std_dev": 0.012957518436208841, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3578504977777778, "avg_duration_per_task_ms": 113715.46888888889, "avg_output_tokens_per_task": 7110.575555555555, "last_run_timestamp": "2026-10-03T08:17:21.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.037271729999999996, "avg_duration_per_task_ms": 168918.11, "avg_output_tokens_per_task": 24042.94, "last_run_timestamp": "2026-10-03T08:36:42.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7466665, "accuracy_std_dev": 0.02854502616189613, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17205141333333332, "avg_duration_per_task_ms": 322261.24, "avg_output_tokens_per_task": 11142.893333333333, "last_run_timestamp": "2026-10-03T08:20:41.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7466421794871795, "accuracy_std_dev": 0.021108433938566915, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.14117064285714287, "avg_duration_per_task_ms": 562123.7032967033, "avg_output_tokens_per_task": 27465.01098901099, "last_run_timestamp": "2026-10-03T16:05:51.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7462312663316583, "accuracy_std_dev": 0.0675814732785547, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.24451848241206028, "avg_duration_per_task_ms": 427951.13567839196, "avg_output_tokens_per_task": 27213.93216080402, "last_run_timestamp": "2026-10-03T16:11:17.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7461904285714286, "accuracy_std_dev": 0.01987481114273, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034756851428571434, "avg_duration_per_task_ms": 433579.28571428574, "avg_output_tokens_per_task": 9627.26857142857, "last_run_timestamp": "2026-08-22T16:09:51.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.745926, "accuracy_std_dev": 0.02095136179610079, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3237732711111111, "avg_duration_per_task_ms": 198551.82222222222, "avg_output_tokens_per_task": 11463.67111111111, "last_run_timestamp": "2026-10-03T08:12:20.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.745926, "accuracy_std_dev": 0.019737876245094522, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.32719486666666664, "avg_duration_per_task_ms": 562449.7333333333, "avg_output_tokens_per_task": 25798.042222222222, "last_run_timestamp": "2026-10-03T16:09:05.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7459258888888889, "accuracy_std_dev": 0.029555403324169362, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07304336, "avg_duration_per_task_ms": 258879.6311111111, "avg_output_tokens_per_task": 9877.288888888888, "last_run_timestamp": "2026-10-03T16:21:25.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7458334999999999, "accuracy_std_dev": 0.02933281723172869, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.28571915, "avg_duration_per_task_ms": 523715.0475, "avg_output_tokens_per_task": 9580.605, "last_run_timestamp": "2026-10-03T16:04:32.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.040383631987350975, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.011250988571428571, "avg_duration_per_task_ms": 1210664.6628571427, "avg_output_tokens_per_task": 11628.308571428572, "last_run_timestamp": "2026-10-03T16:13:56.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7455556666666666, "accuracy_std_dev": 0.02506781419913237, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.3528521766666667, "avg_duration_per_task_ms": 231482.565, "avg_output_tokens_per_task": 10841.498333333333, "last_run_timestamp": "2026-10-03T08:06:43.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7453332, "accuracy_std_dev": 0.019043753132195366, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014976084, "avg_duration_per_task_ms": 412623.44, "avg_output_tokens_per_task": 16235.352, "last_run_timestamp": "2026-10-03T08:13:54.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.03648480000029778, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09990454222222223, "avg_duration_per_task_ms": 432298.65777777776, "avg_output_tokens_per_task": 9726.897777777778, "last_run_timestamp": "2026-10-03T16:05:51.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7451851111111111, "accuracy_std_dev": 0.023471128692475918, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01492189111111111, "avg_duration_per_task_ms": 122706.42666666667, "avg_output_tokens_per_task": 15563.857777777777, "last_run_timestamp": "2026-10-03T13:25:39.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.745, "accuracy_std_dev": 0.0050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.010689059999999998, "avg_duration_per_task_ms": 257202.02, "avg_output_tokens_per_task": 15869.95, "last_run_timestamp": "2026-10-03T08:16:12.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.745, "accuracy_std_dev": 0.015184184024833212, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.055364632500000004, "avg_duration_per_task_ms": 219991, "avg_output_tokens_per_task": 9220.305, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.745, "accuracy_std_dev": 0.046597238036890865, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.12023054166666668, "avg_duration_per_task_ms": 155795.78166666668, "avg_output_tokens_per_task": 9438.261666666667, "last_run_timestamp": "2026-10-03T16:12:33.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.745, "accuracy_std_dev": 0.019649399202011284, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.019549249999999997, "avg_duration_per_task_ms": 83357.6, "avg_output_tokens_per_task": 4168.05, "last_run_timestamp": "2026-10-03T08:30:42.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7448975714285715, "accuracy_std_dev": 0.011805499999999969, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.26889934693877554, "avg_duration_per_task_ms": 528124.887755102, "avg_output_tokens_per_task": 13421.112244897959, "last_run_timestamp": "2026-10-03T08:42:17.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7447619999999999, "accuracy_std_dev": 0.026836107941140986, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3472446428571429, "avg_duration_per_task_ms": 201649.75142857144, "avg_output_tokens_per_task": 11858.474285714286, "last_run_timestamp": "2026-10-03T08:17:06.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.02665859574468085, "avg_duration_per_task_ms": 538486.1063829787, "avg_output_tokens_per_task": 9135.659574468085, "last_run_timestamp": "2026-10-03T08:11:53.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7444445, "accuracy_std_dev": 0.03235589896113743, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007050623333333332, "avg_duration_per_task_ms": 202444.91666666666, "avg_output_tokens_per_task": 4335.703333333333, "last_run_timestamp": "2026-10-03T08:42:41.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7444444444444445, "accuracy_std_dev": 0.028803274848959327, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5183121866666667, "avg_duration_per_task_ms": 226393.7577777778, "avg_output_tokens_per_task": 10022.293333333333, "last_run_timestamp": "2026-10-03T08:15:30.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7444444444444445, "accuracy_std_dev": 0.01987615980186154, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.011055831111111109, "avg_duration_per_task_ms": 226361.5911111111, "avg_output_tokens_per_task": 14386.48, "last_run_timestamp": "2026-10-03T16:11:33.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2319787333333332, "avg_duration_per_task_ms": 98531.56, "avg_output_tokens_per_task": 5524.42, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.34168689999999996, "avg_duration_per_task_ms": 130878.06, "avg_output_tokens_per_task": 7080.526666666667, "last_run_timestamp": "2026-10-03T08:24:43.704Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.02845816509467109, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00811217, "avg_duration_per_task_ms": 250640.82333333333, "avg_output_tokens_per_task": 5287.056666666666, "last_run_timestamp": "2026-10-03T08:42:16.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7441947955056181, "accuracy_std_dev": 0.013929720069329346, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.10463494606741572, "avg_duration_per_task_ms": 943614.8921348314, "avg_output_tokens_per_task": 21227.357303370787, "last_run_timestamp": "2026-09-26T08:24:43.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.744166625, "accuracy_std_dev": 0.029237937922917463, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1196885575, "avg_duration_per_task_ms": 407890.7825, "avg_output_tokens_per_task": 9306.4675, "last_run_timestamp": "2026-10-03T16:14:05.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7437036666666667, "accuracy_std_dev": 0.03863221183876941, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11224634666666668, "avg_duration_per_task_ms": 284243.8644444444, "avg_output_tokens_per_task": 9107.922222222222, "last_run_timestamp": "2026-10-03T16:18:39.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.74338923575419, "accuracy_std_dev": 0.032960776721153426, "run_count": 18, "total_questions": 895, "avg_cost_per_task_usd": 0.11096166927374301, "avg_duration_per_task_ms": 107590.68156424581, "avg_output_tokens_per_task": 8138.489385474861, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02746015, "avg_duration_per_task_ms": 758204.27, "avg_output_tokens_per_task": 13329.06, "last_run_timestamp": "2026-07-18T08:03:38.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7433330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34579436, "avg_duration_per_task_ms": 133736.57, "avg_output_tokens_per_task": 7063.35, "last_run_timestamp": "2026-10-03T08:17:21.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7432883288590605, "accuracy_std_dev": 0.0384259139109129, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.058029342281879205, "avg_duration_per_task_ms": 234207.55369127516, "avg_output_tokens_per_task": 7397.003355704698, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7430303636363637, "accuracy_std_dev": 0.04248479617949489, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.13434207818181818, "avg_duration_per_task_ms": 839557.9927272727, "avg_output_tokens_per_task": 32597.374545454546, "last_run_timestamp": "2026-10-03T16:09:32.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7425475772357722, "accuracy_std_dev": 0.026593049538554243, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.05760138211382114, "avg_duration_per_task_ms": 427993.5081300813, "avg_output_tokens_per_task": 12666.333333333334, "last_run_timestamp": "2026-10-03T08:41:56.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.74250025, "accuracy_std_dev": 0.024874694303397962, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01637653, "avg_duration_per_task_ms": 283609.14, "avg_output_tokens_per_task": 16652.845, "last_run_timestamp": "2026-10-03T08:46:48.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.742500125, "accuracy_std_dev": 0.029802515508080434, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07630773249999999, "avg_duration_per_task_ms": 525311.875, "avg_output_tokens_per_task": 11683.695, "last_run_timestamp": "2026-10-03T16:12:13.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.74239996, "accuracy_std_dev": 0.0339871010313972, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.3750872199999999, "avg_duration_per_task_ms": 202842.4192, "avg_output_tokens_per_task": 11926.788, "last_run_timestamp": "2026-10-03T08:42:18.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.742267824742268, "accuracy_std_dev": 0.022978500000000013, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.011093752577319588, "avg_duration_per_task_ms": 1767717.5051546392, "avg_output_tokens_per_task": 17128.134020618556, "last_run_timestamp": "2026-10-03T08:17:21.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.006285707880227596, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.38184122000000004, "avg_duration_per_task_ms": 149071.25333333333, "avg_output_tokens_per_task": 12794.173333333334, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0028365266666666666, "avg_duration_per_task_ms": 68565.14666666667, "avg_output_tokens_per_task": 11675.026666666667, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7422222222222222, "accuracy_std_dev": 0.023306958640323044, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08909326444444444, "avg_duration_per_task_ms": 804074.62, "avg_output_tokens_per_task": 37512.27111111111, "last_run_timestamp": "2026-10-03T16:15:37.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7422221111111111, "accuracy_std_dev": 0.02012307808597685, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009145446666666668, "avg_duration_per_task_ms": 281815.08222222223, "avg_output_tokens_per_task": 4191.742222222222, "last_run_timestamp": "2026-10-03T08:37:33.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7421122716049383, "accuracy_std_dev": 0.04949233809146621, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.07618280658436215, "avg_duration_per_task_ms": 605963.6255144033, "avg_output_tokens_per_task": 7686.909465020576, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7419176153846153, "accuracy_std_dev": 0.04044592450077292, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.02214324247491639, "avg_duration_per_task_ms": 326050.33277591976, "avg_output_tokens_per_task": 12612.583612040135, "last_run_timestamp": "2026-10-03T08:11:53.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7419047142857143, "accuracy_std_dev": 0.02753696505589889, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.34922034571428573, "avg_duration_per_task_ms": 221709.42857142858, "avg_output_tokens_per_task": 10867.765714285715, "last_run_timestamp": "2026-10-03T08:22:39.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7417787623318386, "accuracy_std_dev": 0.02093398684990749, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.11271321748878924, "avg_duration_per_task_ms": 104918.57623318386, "avg_output_tokens_per_task": 9898.578475336322, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7416667499999999, "accuracy_std_dev": 0.03032220949959782, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08285687, "avg_duration_per_task_ms": 201628.3825, "avg_output_tokens_per_task": 10019.7275, "last_run_timestamp": "2026-10-03T16:14:23.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.741666625, "accuracy_std_dev": 0.015545511144197728, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.00868724, "avg_duration_per_task_ms": 150133.0975, "avg_output_tokens_per_task": 13397.195, "last_run_timestamp": "2026-10-03T16:14:52.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7416665, "accuracy_std_dev": 0.015898969219732467, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011650780000000001, "avg_duration_per_task_ms": 547605.185, "avg_output_tokens_per_task": 5560.365, "last_run_timestamp": "2026-10-03T08:36:39.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7415591841620627, "accuracy_std_dev": 0.038947192468013606, "run_count": 11, "total_questions": 543, "avg_cost_per_task_usd": 0.12226556721915285, "avg_duration_per_task_ms": 510809.66114180477, "avg_output_tokens_per_task": 24893.696132596684, "last_run_timestamp": "2026-10-03T08:18:33.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7414814444444444, "accuracy_std_dev": 0.022172731311285703, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07712953555555555, "avg_duration_per_task_ms": 256540.3088888889, "avg_output_tokens_per_task": 11203.284444444444, "last_run_timestamp": "2026-10-03T16:11:33.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7413600133779265, "accuracy_std_dev": 0.021148441080582975, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06112101672240802, "avg_duration_per_task_ms": 177751.98996655518, "avg_output_tokens_per_task": 7034.244147157191, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.025361580257130863, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.019523156666666666, "avg_duration_per_task_ms": 217853.99333333335, "avg_output_tokens_per_task": 12622.736666666666, "last_run_timestamp": "2026-08-29T16:56:37.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.03430067666687576, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4197743266666667, "avg_duration_per_task_ms": 38304.543333333335, "avg_output_tokens_per_task": 9197.25, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.7407407777777778, "accuracy_std_dev": 0.03670968370812554, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04790690888888889, "avg_duration_per_task_ms": 506951.23333333334, "avg_output_tokens_per_task": 9305.406666666666, "last_run_timestamp": "2026-10-04T00:00:36.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7407406666666667, "accuracy_std_dev": 0.020714235711488657, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07330832000000001, "avg_duration_per_task_ms": 331512.53777777776, "avg_output_tokens_per_task": 12794.486666666666, "last_run_timestamp": "2026-10-03T16:14:15.894Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7405555833333334, "accuracy_std_dev": 0.026311338096146348, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023486161666666668, "avg_duration_per_task_ms": 159438.6, "avg_output_tokens_per_task": 7518.025, "last_run_timestamp": "2026-10-03T16:10:46.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7403703333333334, "accuracy_std_dev": 0.030325141571529955, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3409674066666667, "avg_duration_per_task_ms": 128710.22, "avg_output_tokens_per_task": 5246.528888888889, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7403273080357142, "accuracy_std_dev": 0.029611212101141125, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.029858823660714285, "avg_duration_per_task_ms": 549473.7120535715, "avg_output_tokens_per_task": 14542.716517857143, "last_run_timestamp": "2026-10-04T00:00:28.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400002, "accuracy_std_dev": 0.01264900523203309, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034869184, "avg_duration_per_task_ms": 129959.044, "avg_output_tokens_per_task": 16349.368, "last_run_timestamp": "2026-10-03T08:21:01.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.74, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3455118266666667, "avg_duration_per_task_ms": 100747.32666666666, "avg_output_tokens_per_task": 6984.213333333333, "last_run_timestamp": "2026-10-03T08:20:22.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2696536, "avg_duration_per_task_ms": 310259.6, "avg_output_tokens_per_task": 11435.58, "last_run_timestamp": "2026-07-18T08:04:29.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0077757799999999995, "avg_duration_per_task_ms": 547010.54, "avg_output_tokens_per_task": 14311.64, "last_run_timestamp": "2026-10-03T08:35:48.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05110574, "avg_duration_per_task_ms": 158677.44, "avg_output_tokens_per_task": 8249.06, "last_run_timestamp": "2026-07-18T16:00:09.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.74, "accuracy_std_dev": 0.044555500139339316, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.0935073933333332, "avg_duration_per_task_ms": 295552.94, "avg_output_tokens_per_task": 22860.126666666667, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02561014, "avg_duration_per_task_ms": 147210.62, "avg_output_tokens_per_task": 10787.18, "last_run_timestamp": "2026-07-18T08:04:42.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7399999166666666, "accuracy_std_dev": 0.04127600414982522, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09576565499999999, "avg_duration_per_task_ms": 151409.00166666668, "avg_output_tokens_per_task": 9037.948333333334, "last_run_timestamp": "2026-10-03T16:15:37.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7399998333333334, "accuracy_std_dev": 0.02108185106844802, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.015687136666666667, "avg_duration_per_task_ms": 84185.48333333334, "avg_output_tokens_per_task": 4443.856666666667, "last_run_timestamp": "2026-10-03T08:49:53.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7396624683544304, "accuracy_std_dev": 0.02347519769052809, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.019082217721518985, "avg_duration_per_task_ms": 310670.8101265823, "avg_output_tokens_per_task": 15967.20506329114, "last_run_timestamp": "2026-10-03T08:37:28.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7396296666666666, "accuracy_std_dev": 0.019177150471908547, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.48443006888888884, "avg_duration_per_task_ms": 673286.5122222222, "avg_output_tokens_per_task": 9807.494444444445, "last_run_timestamp": "2026-10-03T08:24:07.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7394443333333334, "accuracy_std_dev": 0.024260716100771285, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023661249999999998, "avg_duration_per_task_ms": 387738.6583333333, "avg_output_tokens_per_task": 12362.963333333333, "last_run_timestamp": "2026-10-03T08:16:24.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.7392593333333333, "accuracy_std_dev": 0.014553267185679581, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015606891111111108, "avg_duration_per_task_ms": 184036.77333333335, "avg_output_tokens_per_task": 4234.36, "last_run_timestamp": "2026-10-03T13:17:20.799Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7391550136986301, "accuracy_std_dev": 0.013576518152268981, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.0136173698630137, "avg_duration_per_task_ms": 57786.109589041094, "avg_output_tokens_per_task": 3732.6301369863013, "last_run_timestamp": "2026-10-03T08:21:31.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7386668, "accuracy_std_dev": 0.009798067838099509, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021031684000000002, "avg_duration_per_task_ms": 190250.892, "avg_output_tokens_per_task": 4221.716, "last_run_timestamp": "2026-10-03T08:46:01.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.03154897514486578, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03057054888888889, "avg_duration_per_task_ms": 269640.66222222225, "avg_output_tokens_per_task": 17038.204444444444, "last_run_timestamp": "2026-10-03T08:35:45.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.017992647486688756, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3103866155555556, "avg_duration_per_task_ms": 191198.25333333333, "avg_output_tokens_per_task": 10879.702222222222, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7383335, "accuracy_std_dev": 0.03261724152193133, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12250923999999998, "avg_duration_per_task_ms": 419416.9975, "avg_output_tokens_per_task": 11580.0775, "last_run_timestamp": "2026-10-03T16:34:12.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7382865983935742, "accuracy_std_dev": 0.027461257286584682, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.07498479919678715, "avg_duration_per_task_ms": 91080.29718875502, "avg_output_tokens_per_task": 4367.955823293173, "last_run_timestamp": "2026-09-26T08:11:26.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7381758445945945, "accuracy_std_dev": 0.015122946755951125, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.020533699324324327, "avg_duration_per_task_ms": 105846.9831081081, "avg_output_tokens_per_task": 3919.314189189189, "last_run_timestamp": "2026-10-03T08:06:57.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.737500125, "accuracy_std_dev": 0.01854046812675925, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.386765285, "avg_duration_per_task_ms": 179778.535, "avg_output_tokens_per_task": 10745.6125, "last_run_timestamp": "2026-10-03T08:40:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.7375, "accuracy_std_dev": 0.016645620047027417, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08416762750000001, "avg_duration_per_task_ms": 304561.8525, "avg_output_tokens_per_task": 13455.715, "last_run_timestamp": "2026-10-03T16:10:32.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7374686954887218, "accuracy_std_dev": 0.031950697455106813, "run_count": 11, "total_questions": 532, "avg_cost_per_task_usd": 0.0578316992481203, "avg_duration_per_task_ms": 124019.84022556391, "avg_output_tokens_per_task": 7638.883458646616, "last_run_timestamp": "2026-10-03T16:12:10.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7374528407310704, "accuracy_std_dev": 0.026611110651845488, "run_count": 23, "total_questions": 1149, "avg_cost_per_task_usd": 0.12236416971279374, "avg_duration_per_task_ms": 174480.13751087902, "avg_output_tokens_per_task": 9112.760661444734, "last_run_timestamp": "2026-10-03T16:13:53.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7373735454545455, "accuracy_std_dev": 0.022590007983343003, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.0463935, "avg_duration_per_task_ms": 782908.3939393939, "avg_output_tokens_per_task": 14148.70202020202, "last_run_timestamp": "2026-10-04T00:00:36.967Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7371428571428571, "accuracy_std_dev": 0.033210782885032, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4983071857142858, "avg_duration_per_task_ms": 191159.05714285714, "avg_output_tokens_per_task": 9902.92, "last_run_timestamp": "2026-10-03T08:21:29.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.37755702999999996, "avg_duration_per_task_ms": 287090.13, "avg_output_tokens_per_task": 9995.37, "last_run_timestamp": "2026-10-03T08:10:45.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2879983000000002, "avg_duration_per_task_ms": 310547.73, "avg_output_tokens_per_task": 22913.4, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06833741, "avg_duration_per_task_ms": 266166.56, "avg_output_tokens_per_task": 13745.94, "last_run_timestamp": "2026-10-03T08:19:53.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7366668333333333, "accuracy_std_dev": 0.023333261905247794, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.35079288666666664, "avg_duration_per_task_ms": 89188.23666666666, "avg_output_tokens_per_task": 6914.113333333334, "last_run_timestamp": "2026-10-03T08:12:46.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7366668, "accuracy_std_dev": 0.022900761755015906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.063241744, "avg_duration_per_task_ms": 1530537.112, "avg_output_tokens_per_task": 25222.716, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7366667499999999, "accuracy_std_dev": 0.0328295429291591, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10887684250000002, "avg_duration_per_task_ms": 323590.875, "avg_output_tokens_per_task": 18425.0325, "last_run_timestamp": "2026-10-03T08:19:27.960Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.736666625, "accuracy_std_dev": 0.03915774721219767, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0416237425, "avg_duration_per_task_ms": 215980.8525, "avg_output_tokens_per_task": 20025.32, "last_run_timestamp": "2026-10-03T08:21:01.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7366665, "accuracy_std_dev": 0.023333357143440252, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.019450683333333333, "avg_duration_per_task_ms": 216938.31, "avg_output_tokens_per_task": 4564.706666666667, "last_run_timestamp": "2026-10-03T08:34:35.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7365900057471265, "accuracy_std_dev": 0.030044845419567656, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.09514021264367815, "avg_duration_per_task_ms": 383890.3362068966, "avg_output_tokens_per_task": 11244.488505747126, "last_run_timestamp": "2026-10-03T16:13:47.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7361110833333333, "accuracy_std_dev": 0.03144161010099605, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.030371235, "avg_duration_per_task_ms": 428092.15, "avg_output_tokens_per_task": 13168.441666666668, "last_run_timestamp": "2026-10-03T08:16:37.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.736, "accuracy_std_dev": 0.01717892766152766, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.23393872400000001, "avg_duration_per_task_ms": 250693.228, "avg_output_tokens_per_task": 12185.72, "last_run_timestamp": "2026-10-03T08:11:53.179Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.736, "accuracy_std_dev": 0.006798758063058283, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.091687124, "avg_duration_per_task_ms": 467869.472, "avg_output_tokens_per_task": 12583.56, "last_run_timestamp": "2026-10-03T08:09:52.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7358334999999999, "accuracy_std_dev": 0.025097105854062156, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015653559999999997, "avg_duration_per_task_ms": 644029.88, "avg_output_tokens_per_task": 23956.73, "last_run_timestamp": "2026-10-03T08:19:27.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.73583325, "accuracy_std_dev": 0.04026825845424036, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0081671125, "avg_duration_per_task_ms": 333111.9, "avg_output_tokens_per_task": 15655.8425, "last_run_timestamp": "2026-10-03T08:38:11.814Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13620342666666665, "avg_duration_per_task_ms": 101042.81333333334, "avg_output_tokens_per_task": 8769.966666666667, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7355554444444444, "accuracy_std_dev": 0.012171531247693556, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3527382688888889, "avg_duration_per_task_ms": 87063.18222222223, "avg_output_tokens_per_task": 7051.828888888889, "last_run_timestamp": "2026-10-03T08:18:17.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7355554166666667, "accuracy_std_dev": 0.0214879236838522, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.007812348333333334, "avg_duration_per_task_ms": 338040.58666666667, "avg_output_tokens_per_task": 14032.848333333333, "last_run_timestamp": "2026-10-03T08:21:29.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.735194730964467, "accuracy_std_dev": 0.02636439034355808, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.11292960406091371, "avg_duration_per_task_ms": 484722.62436548225, "avg_output_tokens_per_task": 29978.527918781725, "last_run_timestamp": "2026-10-03T08:25:13.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7351928099173552, "accuracy_std_dev": 0.026445667029780136, "run_count": 10, "total_questions": 484, "avg_cost_per_task_usd": 0.019751508264462807, "avg_duration_per_task_ms": 93530.47314049587, "avg_output_tokens_per_task": 6382.072314049587, "last_run_timestamp": "2026-10-03T16:12:29.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7349998750000001, "accuracy_std_dev": 0.029767504188449773, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1274803, "avg_duration_per_task_ms": 258225.3975, "avg_output_tokens_per_task": 10070.7475, "last_run_timestamp": "2026-10-03T16:12:24.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7349997500000001, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12295462999999998, "avg_duration_per_task_ms": 89986.35, "avg_output_tokens_per_task": 23350.75, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7346666000000001, "accuracy_std_dev": 0.025785270559759498, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.382410584, "avg_duration_per_task_ms": 202893.328, "avg_output_tokens_per_task": 8639.172, "last_run_timestamp": "2026-10-03T08:18:05.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7345953452115812, "accuracy_std_dev": 0.020421322845629582, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.01083965478841871, "avg_duration_per_task_ms": 494548.15144766145, "avg_output_tokens_per_task": 14266.481069042316, "last_run_timestamp": "2026-10-03T08:38:09.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7343749866071428, "accuracy_std_dev": 0.03726027594412176, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.08364443973214285, "avg_duration_per_task_ms": 526591.8638392857, "avg_output_tokens_per_task": 14298.44642857143, "last_run_timestamp": "2026-10-03T16:14:31.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7342569118387909, "accuracy_std_dev": 0.03174336773937983, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.07517244332493701, "avg_duration_per_task_ms": 212080.59697733, "avg_output_tokens_per_task": 8222.277078085643, "last_run_timestamp": "2026-10-03T16:14:02.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.7340742222222222, "accuracy_std_dev": 0.04842648211642921, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07961510222222222, "avg_duration_per_task_ms": 198839.9311111111, "avg_output_tokens_per_task": 8248.502222222221, "last_run_timestamp": "2026-09-19T08:12:24.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7340741111111111, "accuracy_std_dev": 0.04903445388804455, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.00916024, "avg_duration_per_task_ms": 334894.9577777778, "avg_output_tokens_per_task": 15526.06, "last_run_timestamp": "2026-10-03T08:11:53.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.73375, "accuracy_std_dev": 0.03190438406003161, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07429103249999999, "avg_duration_per_task_ms": 245485.365, "avg_output_tokens_per_task": 11375.32, "last_run_timestamp": "2026-10-03T16:17:46.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7333336666666668, "accuracy_std_dev": 0.03771236166328251, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.44470873333333333, "avg_duration_per_task_ms": 130557.96, "avg_output_tokens_per_task": 20380.893333333333, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7333334444444444, "accuracy_std_dev": 0.020846304112139443, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06912064444444445, "avg_duration_per_task_ms": 144871.33777777778, "avg_output_tokens_per_task": 9759.733333333334, "last_run_timestamp": "2026-10-03T16:19:07.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.73333325, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025873325, "avg_duration_per_task_ms": 147169.53, "avg_output_tokens_per_task": 18134.845, "last_run_timestamp": "2026-10-03T08:34:36.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.73333325, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17039939499999998, "avg_duration_per_task_ms": 114141.445, "avg_output_tokens_per_task": 5083.04, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.73333325, "accuracy_std_dev": 0.02815296839922036, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06546306166666667, "avg_duration_per_task_ms": 280345.87666666665, "avg_output_tokens_per_task": 9846.026666666667, "last_run_timestamp": "2026-10-03T16:07:50.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04628359, "avg_duration_per_task_ms": 154349.4, "avg_output_tokens_per_task": 38415.97, "last_run_timestamp": "2026-10-03T08:10:45.808Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7331108173719375, "accuracy_std_dev": 0.03143352561191731, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.07030102004454343, "avg_duration_per_task_ms": 234100.18485523385, "avg_output_tokens_per_task": 11575.886414253897, "last_run_timestamp": "2026-10-03T16:23:41.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02503509736024079, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06745117142857142, "avg_duration_per_task_ms": 76562.92857142857, "avg_output_tokens_per_task": 8496.594285714285, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7327273636363636, "accuracy_std_dev": 0.10615232467139141, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.14514397454545452, "avg_duration_per_task_ms": 616412.5018181818, "avg_output_tokens_per_task": 27320.474545454545, "last_run_timestamp": "2026-10-03T08:15:42.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7325925555555556, "accuracy_std_dev": 0.02443313150262758, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.3673924444444444, "avg_duration_per_task_ms": 230445.1311111111, "avg_output_tokens_per_task": 10853.041111111112, "last_run_timestamp": "2026-10-03T08:12:59.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.732381, "accuracy_std_dev": 0.026530205265481322, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07102172, "avg_duration_per_task_ms": 255607.41142857142, "avg_output_tokens_per_task": 7988.711428571429, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7320002, "accuracy_std_dev": 0.03435759143129796, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.016945259999999997, "avg_duration_per_task_ms": 153216.504, "avg_output_tokens_per_task": 14926.164, "last_run_timestamp": "2026-10-03T08:34:50.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.73166675, "accuracy_std_dev": 0.01280170344866263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021476995, "avg_duration_per_task_ms": 208483.135, "avg_output_tokens_per_task": 8547.82, "last_run_timestamp": "2026-10-03T16:05:47.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73166675, "accuracy_std_dev": 0.015184321229067165, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.29879598, "avg_duration_per_task_ms": 177541.565, "avg_output_tokens_per_task": 10870.785, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7315437382550335, "accuracy_std_dev": 0.03437878873504547, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.014048604026845638, "avg_duration_per_task_ms": 181440.27516778524, "avg_output_tokens_per_task": 3805.51677852349, "last_run_timestamp": "2026-10-03T08:39:48.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7315152727272727, "accuracy_std_dev": 0.033162046414239345, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009558798181818184, "avg_duration_per_task_ms": 364447.56727272726, "avg_output_tokens_per_task": 14249.385454545454, "last_run_timestamp": "2026-10-03T08:10:45.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7311827741935484, "accuracy_std_dev": 0.030765855212556675, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.13947222983870966, "avg_duration_per_task_ms": 675891.3467741936, "avg_output_tokens_per_task": 27076.72177419355, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7311113333333332, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10536965333333334, "avg_duration_per_task_ms": 88766.42666666667, "avg_output_tokens_per_task": 4017.0533333333333, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7311110833333333, "accuracy_std_dev": 0.02884609437300635, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02860928166666667, "avg_duration_per_task_ms": 200182.31333333332, "avg_output_tokens_per_task": 9301.73, "last_run_timestamp": "2026-10-03T16:24:32.945Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.022662199996175744, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3988358066666667, "avg_duration_per_task_ms": 172929.82666666666, "avg_output_tokens_per_task": 27148.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7308334999999999, "accuracy_std_dev": 0.023731723962872993, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.043652735000000005, "avg_duration_per_task_ms": 134583.575, "avg_output_tokens_per_task": 27937.025, "last_run_timestamp": "2026-10-03T08:15:00.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7308334999999999, "accuracy_std_dev": 0.018162178345396796, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.064435845, "avg_duration_per_task_ms": 500666.49, "avg_output_tokens_per_task": 9013.95, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.730833375, "accuracy_std_dev": 0.03178700271705992, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04844423749999999, "avg_duration_per_task_ms": 358219.625, "avg_output_tokens_per_task": 10201.3225, "last_run_timestamp": "2026-10-03T16:15:19.157Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7305555, "accuracy_std_dev": 0.02316726484899186, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07086156666666667, "avg_duration_per_task_ms": 634585.7766666666, "avg_output_tokens_per_task": 8337.83, "last_run_timestamp": "2026-10-03T16:16:13.260Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7303030909090908, "accuracy_std_dev": 0.037644147929729685, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05068959090909091, "avg_duration_per_task_ms": 151393.85454545455, "avg_output_tokens_per_task": 8568.572727272727, "last_run_timestamp": "2026-10-03T16:21:53.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7300587002518891, "accuracy_std_dev": 0.027075268354798905, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.37323221158690173, "avg_duration_per_task_ms": 401791.3123425693, "avg_output_tokens_per_task": 9265.304785894206, "last_run_timestamp": "2026-10-03T08:39:26.414Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.7300002499999999, "accuracy_std_dev": 0.024037135622563252, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01993718, "avg_duration_per_task_ms": 483493.9275, "avg_output_tokens_per_task": 14630.11, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.73, "accuracy_std_dev": 0.036666636365138984, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3388476266666666, "avg_duration_per_task_ms": 404706.78, "avg_output_tokens_per_task": 9319.38, "last_run_timestamp": "2026-10-03T08:15:30.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.7299997500000001, "accuracy_std_dev": 0.02768883649934574, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04588337249999999, "avg_duration_per_task_ms": 177085.435, "avg_output_tokens_per_task": 8314.0825, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7299516521739131, "accuracy_std_dev": 0.017388525374334593, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.020714037681159418, "avg_duration_per_task_ms": 279014.0231884058, "avg_output_tokens_per_task": 10794.56811594203, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7295238571428572, "accuracy_std_dev": 0.034215143421598336, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.3555786895238095, "avg_duration_per_task_ms": 206423.8238095238, "avg_output_tokens_per_task": 10296.156190476191, "last_run_timestamp": "2026-10-03T08:16:52.730Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7294444166666667, "accuracy_std_dev": 0.02379831213713252, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02310246, "avg_duration_per_task_ms": 89720.39666666667, "avg_output_tokens_per_task": 7429.111666666667, "last_run_timestamp": "2026-10-03T16:13:58.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7293334, "accuracy_std_dev": 0.013727396775791089, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.21736060000000001, "avg_duration_per_task_ms": 395127.28, "avg_output_tokens_per_task": 9443.484, "last_run_timestamp": "2026-10-03T08:37:59.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7293186532846716, "accuracy_std_dev": 0.029832182888078592, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.042413810218978096, "avg_duration_per_task_ms": 231320.98175182482, "avg_output_tokens_per_task": 9945.337591240876, "last_run_timestamp": "2026-10-03T16:16:50.387Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.729167, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.044939625000000004, "avg_duration_per_task_ms": 321431, "avg_output_tokens_per_task": 27916.854166666668, "last_run_timestamp": "2026-07-11T08:03:26.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7290600769230768, "accuracy_std_dev": 0.010018378923258995, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.1270635282051282, "avg_duration_per_task_ms": 193981.8564102564, "avg_output_tokens_per_task": 10184.36923076923, "last_run_timestamp": "2026-10-03T16:07:34.513Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.728889, "accuracy_std_dev": 0.052305021830285715, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024965873333333333, "avg_duration_per_task_ms": 476087.61333333334, "avg_output_tokens_per_task": 18580.853333333333, "last_run_timestamp": "2026-10-03T08:29:04.085Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7284848181818182, "accuracy_std_dev": 0.02815423218248235, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07946297454545455, "avg_duration_per_task_ms": 222298.18181818182, "avg_output_tokens_per_task": 8681.263636363636, "last_run_timestamp": "2026-10-03T16:14:07.111Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.72833325, "accuracy_std_dev": 0.022298511411919422, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.044024975, "avg_duration_per_task_ms": 136603.385, "avg_output_tokens_per_task": 25927.71, "last_run_timestamp": "2026-10-03T08:49:47.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7277779166666666, "accuracy_std_dev": 0.032413125105061814, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.022190256666666672, "avg_duration_per_task_ms": 117748.965, "avg_output_tokens_per_task": 6863.575, "last_run_timestamp": "2026-10-03T16:15:50.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7275785560538117, "accuracy_std_dev": 0.09500029906437658, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.07257157174887892, "avg_duration_per_task_ms": 157267.63677130046, "avg_output_tokens_per_task": 14011.852017937219, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.72750025, "accuracy_std_dev": 0.014790298059454364, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013181285000000001, "avg_duration_per_task_ms": 37840.265, "avg_output_tokens_per_task": 3828.01, "last_run_timestamp": "2026-10-03T08:06:57.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7275, "accuracy_std_dev": 0.011273272229038026, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.25819418250000004, "avg_duration_per_task_ms": 292629.7475, "avg_output_tokens_per_task": 10249.3025, "last_run_timestamp": "2026-09-26T08:16:17.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7273567338262477, "accuracy_std_dev": 0.034097962965258875, "run_count": 11, "total_questions": 541, "avg_cost_per_task_usd": 0.10647174121996304, "avg_duration_per_task_ms": 340266.1959334566, "avg_output_tokens_per_task": 15129.060998151572, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7271506693548386, "accuracy_std_dev": 0.030421326219611147, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.13253313306451614, "avg_duration_per_task_ms": 725346.5120967742, "avg_output_tokens_per_task": 29741.665322580644, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7271427142857143, "accuracy_std_dev": 0.03945770950094827, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.00689044, "avg_duration_per_task_ms": 394490.1285714286, "avg_output_tokens_per_task": 14238.282857142856, "last_run_timestamp": "2026-10-03T08:19:00.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7270693489932887, "accuracy_std_dev": 0.056280655844911175, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.2920954026845638, "avg_duration_per_task_ms": 417994.4194630872, "avg_output_tokens_per_task": 10865.342281879195, "last_run_timestamp": "2026-10-03T08:36:28.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11300302000000001, "avg_duration_per_task_ms": 353321.78, "avg_output_tokens_per_task": 3658.14, "last_run_timestamp": "2026-10-03T00:00:06.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7266667272727273, "accuracy_std_dev": 0.03959384198697136, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1237469890909091, "avg_duration_per_task_ms": 184460.7309090909, "avg_output_tokens_per_task": 12090.834545454545, "last_run_timestamp": "2026-10-03T16:12:35.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.050625568127402036, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17683866, "avg_duration_per_task_ms": 285023.12333333335, "avg_output_tokens_per_task": 10017.94, "last_run_timestamp": "2026-10-03T08:23:53.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7266666363636364, "accuracy_std_dev": 0.028284335530700075, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03399551090909091, "avg_duration_per_task_ms": 161533.11272727273, "avg_output_tokens_per_task": 8657.467272727272, "last_run_timestamp": "2026-10-03T16:08:50.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.726666625, "accuracy_std_dev": 0.03000002777889339, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034730545, "avg_duration_per_task_ms": 333782.2075, "avg_output_tokens_per_task": 8351.1, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7266666, "accuracy_std_dev": 0.017888692893557094, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.46400111200000005, "avg_duration_per_task_ms": 797376.568, "avg_output_tokens_per_task": 9519.38, "last_run_timestamp": "2026-10-03T08:14:34.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.7266665, "accuracy_std_dev": 0.021343838560577626, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.050907680000000004, "avg_duration_per_task_ms": 398195.5025, "avg_output_tokens_per_task": 14890.9375, "last_run_timestamp": "2026-10-04T00:00:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7266665, "accuracy_std_dev": 0.033665214491370776, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027464944999999998, "avg_duration_per_task_ms": 167583.24, "avg_output_tokens_per_task": 16249.64, "last_run_timestamp": "2026-10-03T08:11:53.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7266665, "accuracy_std_dev": 0.03162256578378802, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.325251515, "avg_duration_per_task_ms": 113433.705, "avg_output_tokens_per_task": 10925.445, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7264311717171718, "accuracy_std_dev": 0.023522929859766603, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04122346464646465, "avg_duration_per_task_ms": 213769.9797979798, "avg_output_tokens_per_task": 25500.383838383837, "last_run_timestamp": "2026-10-03T08:37:32.589Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7260604545454545, "accuracy_std_dev": 0.023860693807498716, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07064769636363637, "avg_duration_per_task_ms": 199404.25636363638, "avg_output_tokens_per_task": 8425.005454545455, "last_run_timestamp": "2026-10-03T16:25:05.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7259357914438503, "accuracy_std_dev": 0.03183481835768294, "run_count": 15, "total_questions": 748, "avg_cost_per_task_usd": 0.1275639679144385, "avg_duration_per_task_ms": 104736.3101604278, "avg_output_tokens_per_task": 9200.958556149733, "last_run_timestamp": "2026-10-03T16:09:05.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.725926, "accuracy_std_dev": 0.02656370968487314, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05504480444444444, "avg_duration_per_task_ms": 317296.9066666667, "avg_output_tokens_per_task": 11442.59111111111, "last_run_timestamp": "2026-10-03T16:05:51.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7255555, "accuracy_std_dev": 0.026643554800676796, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07362210166666668, "avg_duration_per_task_ms": 249674.02666666667, "avg_output_tokens_per_task": 8499.056666666667, "last_run_timestamp": "2026-10-03T16:12:07.465Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7255554166666667, "accuracy_std_dev": 0.021227793045354233, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07783504166666667, "avg_duration_per_task_ms": 257422.64666666667, "avg_output_tokens_per_task": 9069.785, "last_run_timestamp": "2026-10-03T16:06:47.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.725244179916318, "accuracy_std_dev": 0.026125398745282327, "run_count": 5, "total_questions": 239, "avg_cost_per_task_usd": 0.13789057740585775, "avg_duration_per_task_ms": 590927.3305439331, "avg_output_tokens_per_task": 27054.07949790795, "last_run_timestamp": "2026-10-03T16:07:34.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7248775772058824, "accuracy_std_dev": 0.06361727543977472, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.1465529356617647, "avg_duration_per_task_ms": 1298187.9650735294, "avg_output_tokens_per_task": 27211.007352941175, "last_run_timestamp": "2026-10-03T16:07:44.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7247618571428571, "accuracy_std_dev": 0.03947509103012603, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06044658285714286, "avg_duration_per_task_ms": 560702.9542857143, "avg_output_tokens_per_task": 14126.951428571429, "last_run_timestamp": "2026-08-29T17:14:29.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7247618571428571, "accuracy_std_dev": 0.04843106328860957, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04376539714285715, "avg_duration_per_task_ms": 289052.61714285717, "avg_output_tokens_per_task": 14151.488571428572, "last_run_timestamp": "2026-10-03T08:24:26.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7244445555555555, "accuracy_std_dev": 0.024944550855006874, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05747211333333333, "avg_duration_per_task_ms": 311689.84222222224, "avg_output_tokens_per_task": 9750.984444444444, "last_run_timestamp": "2026-10-04T00:00:36.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7237036666666666, "accuracy_std_dev": 0.03501115714867033, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3328492511111112, "avg_duration_per_task_ms": 204813.59333333332, "avg_output_tokens_per_task": 10311.322222222223, "last_run_timestamp": "2026-10-03T08:11:53.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7237035555555555, "accuracy_std_dev": 0.03371788875396401, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.29463488, "avg_duration_per_task_ms": 344302.6911111111, "avg_output_tokens_per_task": 9396.468888888889, "last_run_timestamp": "2026-10-03T08:36:28.986Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7235555333333333, "accuracy_std_dev": 0.036907479386282784, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.05777671999999999, "avg_duration_per_task_ms": 117310.35733333333, "avg_output_tokens_per_task": 8612.536, "last_run_timestamp": "2026-10-03T16:13:38.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.723445004784689, "accuracy_std_dev": 0.03240803794328395, "run_count": 21, "total_questions": 1045, "avg_cost_per_task_usd": 0.1143725071770335, "avg_duration_per_task_ms": 143879.64210526316, "avg_output_tokens_per_task": 8375.931100478469, "last_run_timestamp": "2026-10-03T16:12:20.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.7233335, "accuracy_std_dev": 0.016666800000300002, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.052515885000000005, "avg_duration_per_task_ms": 281245.84, "avg_output_tokens_per_task": 8472.745, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.72333325, "accuracy_std_dev": 0.01374358436462263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011429815, "avg_duration_per_task_ms": 101515.345, "avg_output_tokens_per_task": 5183.985, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12502769, "avg_duration_per_task_ms": 371456.84, "avg_output_tokens_per_task": 16328.88, "last_run_timestamp": "2026-10-03T16:15:26.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7226666, "accuracy_std_dev": 0.02619600308902105, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.065411432, "avg_duration_per_task_ms": 217655.416, "avg_output_tokens_per_task": 9279.2, "last_run_timestamp": "2026-10-03T16:12:07.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.7225, "accuracy_std_dev": 0.02665364265442906, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05203516, "avg_duration_per_task_ms": 275661.6275, "avg_output_tokens_per_task": 8631.19, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7224997500000001, "accuracy_std_dev": 0.03430864201170749, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0677995325, "avg_duration_per_task_ms": 399563.9925, "avg_output_tokens_per_task": 11360.565, "last_run_timestamp": "2026-10-03T16:10:02.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.031427125187575726, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009912693333333332, "avg_duration_per_task_ms": 115398.83333333333, "avg_output_tokens_per_task": 5230.286666666667, "last_run_timestamp": "2026-10-03T08:11:53.182Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7222221493624772, "accuracy_std_dev": 0.036194051658678325, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.10862861566484516, "avg_duration_per_task_ms": 283523.6338797814, "avg_output_tokens_per_task": 10296.255009107468, "last_run_timestamp": "2026-10-03T16:20:53.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7222221428571428, "accuracy_std_dev": 0.024312166707227047, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.046834047619047615, "avg_duration_per_task_ms": 58108.6462585034, "avg_output_tokens_per_task": 13243.625850340137, "last_run_timestamp": "2026-09-29T21:00:05.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.722222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019033145833333334, "avg_duration_per_task_ms": 65993.54166666667, "avg_output_tokens_per_task": 3355.9791666666665, "last_run_timestamp": "2026-09-12T08:12:22.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7207407777777778, "accuracy_std_dev": 0.017340394259120186, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10771476888888888, "avg_duration_per_task_ms": 342540.08444444445, "avg_output_tokens_per_task": 15143.33111111111, "last_run_timestamp": "2026-10-04T00:00:34.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7207407777777778, "accuracy_std_dev": 0.01790079352541406, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04994019555555556, "avg_duration_per_task_ms": 379059.3222222222, "avg_output_tokens_per_task": 9310.02, "last_run_timestamp": "2026-10-03T16:12:08.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7207406666666667, "accuracy_std_dev": 0.027477417091131398, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3540747222222222, "avg_duration_per_task_ms": 237714.0088888889, "avg_output_tokens_per_task": 10830.737777777778, "last_run_timestamp": "2026-10-03T08:11:26.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7207406666666667, "accuracy_std_dev": 0.0246344746014379, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05808530888888888, "avg_duration_per_task_ms": 328138.63333333336, "avg_output_tokens_per_task": 9714.76888888889, "last_run_timestamp": "2026-10-04T00:00:28.270Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7205384141414142, "accuracy_std_dev": 0.051112796809483045, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 1.6761988181818182, "avg_duration_per_task_ms": 313695.66161616164, "avg_output_tokens_per_task": 23404.97474747475, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7202819926470587, "accuracy_std_dev": 0.041371140936807896, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.02139444301470588, "avg_duration_per_task_ms": 101681.90625, "avg_output_tokens_per_task": 6783.099264705882, "last_run_timestamp": "2026-10-03T16:04:32.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03553636, "avg_duration_per_task_ms": 263015.66, "avg_output_tokens_per_task": 14384.46, "last_run_timestamp": "2026-07-18T16:00:09.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.72, "accuracy_std_dev": 0.021081956479416224, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03251169, "avg_duration_per_task_ms": 105065.11, "avg_output_tokens_per_task": 22739.925, "last_run_timestamp": "2026-10-03T09:03:15.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.72, "accuracy_std_dev": 0.03231762505197435, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.36211610499999997, "avg_duration_per_task_ms": 150104.23, "avg_output_tokens_per_task": 23123.4, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.72, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.037352619999999996, "avg_duration_per_task_ms": 206767.2, "avg_output_tokens_per_task": 11957.14, "last_run_timestamp": "2026-09-26T08:10:43.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7199996666666667, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09322211333333333, "avg_duration_per_task_ms": 149908.11333333334, "avg_output_tokens_per_task": 49346.81333333333, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7194444166666667, "accuracy_std_dev": 0.03371537501511324, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03395878166666666, "avg_duration_per_task_ms": 181860.895, "avg_output_tokens_per_task": 9239.94, "last_run_timestamp": "2026-10-03T16:08:50.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7193332, "accuracy_std_dev": 0.03143937799893632, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045152332, "avg_duration_per_task_ms": 326196.928, "avg_output_tokens_per_task": 13427.836, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7193332, "accuracy_std_dev": 0.02332387046268266, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011747176000000002, "avg_duration_per_task_ms": 276612.664, "avg_output_tokens_per_task": 17492.112, "last_run_timestamp": "2026-10-03T08:34:32.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7192592222222223, "accuracy_std_dev": 0.019485908525437762, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02609890888888889, "avg_duration_per_task_ms": 230277.4688888889, "avg_output_tokens_per_task": 10113.222222222223, "last_run_timestamp": "2026-10-03T16:12:13.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7191666250000001, "accuracy_std_dev": 0.02633896432539395, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.300338065, "avg_duration_per_task_ms": 406405.855, "avg_output_tokens_per_task": 9208.345, "last_run_timestamp": "2026-10-03T08:11:53.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7191665, "accuracy_std_dev": 0.008291511728870679, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.046859950000000004, "avg_duration_per_task_ms": 203049.32, "avg_output_tokens_per_task": 32440.35, "last_run_timestamp": "2026-10-03T08:38:29.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.7188888333333333, "accuracy_std_dev": 0.026504201858175022, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3197561666666667, "avg_duration_per_task_ms": 193449.66, "avg_output_tokens_per_task": 11623.543333333333, "last_run_timestamp": "2026-10-03T08:06:43.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7188888333333333, "accuracy_std_dev": 0.03297943429581869, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11799081000000002, "avg_duration_per_task_ms": 673829.7433333333, "avg_output_tokens_per_task": 29526.56, "last_run_timestamp": "2026-10-03T08:37:23.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7188551919191919, "accuracy_std_dev": 0.027525589315898408, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.29904142424242425, "avg_duration_per_task_ms": 154139.7323232323, "avg_output_tokens_per_task": 5036.570707070707, "last_run_timestamp": "2026-10-01T19:05:12.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7186897976878613, "accuracy_std_dev": 0.027015736117292515, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.02169187283236994, "avg_duration_per_task_ms": 314303.23988439306, "avg_output_tokens_per_task": 7575.85549132948, "last_run_timestamp": "2026-08-29T16:48:46.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.718667, "accuracy_std_dev": 0.015999999999999966, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012890876, "avg_duration_per_task_ms": 931505.22, "avg_output_tokens_per_task": 16944.504, "last_run_timestamp": "2026-10-01T09:55:07.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7183906609195403, "accuracy_std_dev": 0.025974738950584678, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.06343264655172413, "avg_duration_per_task_ms": 497586.9425287356, "avg_output_tokens_per_task": 14530.419540229885, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7183333749999999, "accuracy_std_dev": 0.031578880513475675, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12769575749999998, "avg_duration_per_task_ms": 325285.93, "avg_output_tokens_per_task": 10098.865, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.71833325, "accuracy_std_dev": 0.022298399296530245, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.039712805, "avg_duration_per_task_ms": 138975.945, "avg_output_tokens_per_task": 15594.74, "last_run_timestamp": "2026-10-03T08:38:07.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7180392352941176, "accuracy_std_dev": 0.014599556014721458, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.037821898823529404, "avg_duration_per_task_ms": 192987.32, "avg_output_tokens_per_task": 9953.656470588236, "last_run_timestamp": "2026-10-03T16:15:22.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7175757272727272, "accuracy_std_dev": 0.03698453502624358, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10178546363636364, "avg_duration_per_task_ms": 116835.80727272727, "avg_output_tokens_per_task": 9163.8, "last_run_timestamp": "2026-10-03T16:12:20.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.7170371111111111, "accuracy_std_dev": 0.016956299841680886, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12728777111111111, "avg_duration_per_task_ms": 101323.44222222222, "avg_output_tokens_per_task": 3885.56, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015077799999999999, "avg_duration_per_task_ms": 476878.72, "avg_output_tokens_per_task": 26097.54, "last_run_timestamp": "2026-09-28T09:00:02.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06700715, "avg_duration_per_task_ms": 992682.5, "avg_output_tokens_per_task": 40930.55, "last_run_timestamp": "2026-10-03T08:22:39.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7163635454545455, "accuracy_std_dev": 0.14986481732266926, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06114290181818182, "avg_duration_per_task_ms": 168295.4890909091, "avg_output_tokens_per_task": 20095.172727272726, "last_run_timestamp": "2026-10-03T16:07:44.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.7162161756756756, "accuracy_std_dev": 0.0136238728300322, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.0890948108108108, "avg_duration_per_task_ms": 1143483.75, "avg_output_tokens_per_task": 17745.06081081081, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.040855070557058414, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012743326666666667, "avg_duration_per_task_ms": 1224143.2066666668, "avg_output_tokens_per_task": 18669.653333333332, "last_run_timestamp": "2026-08-29T09:20:05.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7155553333333334, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13937678, "avg_duration_per_task_ms": 263819.3466666667, "avg_output_tokens_per_task": 10311.9, "last_run_timestamp": "2026-10-03T16:18:50.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7154882272727273, "accuracy_std_dev": 0.03158570170163035, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.06011247222222223, "avg_duration_per_task_ms": 488450.73484848486, "avg_output_tokens_per_task": 14170.416666666666, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.715488212121212, "accuracy_std_dev": 0.013403360380889571, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04391666666666666, "avg_duration_per_task_ms": 58944.35858585859, "avg_output_tokens_per_task": 8110.530303030303, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.715238, "accuracy_std_dev": 0.036811758695907394, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0628034857142857, "avg_duration_per_task_ms": 75005.29714285715, "avg_output_tokens_per_task": 9080.965714285714, "last_run_timestamp": "2026-09-26T08:24:48.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7151513636363637, "accuracy_std_dev": 0.042553816410993, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09130638, "avg_duration_per_task_ms": 220131.52363636365, "avg_output_tokens_per_task": 8503.710909090909, "last_run_timestamp": "2026-10-03T16:13:56.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.715, "accuracy_std_dev": 0.029202378139459823, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09407665500000001, "avg_duration_per_task_ms": 83159.84, "avg_output_tokens_per_task": 3660.855, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7148146666666667, "accuracy_std_dev": 0.031075936024161484, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09082762666666666, "avg_duration_per_task_ms": 156907.88888888888, "avg_output_tokens_per_task": 48272.70222222222, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7145455454545454, "accuracy_std_dev": 0.03220115495208104, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11228732, "avg_duration_per_task_ms": 99294.4490909091, "avg_output_tokens_per_task": 9738.792727272727, "last_run_timestamp": "2026-10-03T16:12:20.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.71416675, "accuracy_std_dev": 0.03290334118577473, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0400061, "avg_duration_per_task_ms": 354559.8625, "avg_output_tokens_per_task": 9774.86, "last_run_timestamp": "2026-10-03T16:14:22.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.714166625, "accuracy_std_dev": 0.0311691275781722, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0320175475, "avg_duration_per_task_ms": 366948.7475, "avg_output_tokens_per_task": 5303.03, "last_run_timestamp": "2026-10-03T08:46:44.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05460591333333333, "avg_duration_per_task_ms": 289287.8333333333, "avg_output_tokens_per_task": 8799.16, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0723164, "avg_duration_per_task_ms": 346522.38, "avg_output_tokens_per_task": 59152.72, "last_run_timestamp": "2026-09-12T16:00:24.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.7130950357142857, "accuracy_std_dev": 0.06532407345092787, "run_count": 3, "total_questions": 140, "avg_cost_per_task_usd": 0.18180849999999998, "avg_duration_per_task_ms": 300783.1928571428, "avg_output_tokens_per_task": 20383.22857142857, "last_run_timestamp": "2026-10-03T16:20:06.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7123811428571428, "accuracy_std_dev": 0.0279293098601685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04820127142857143, "avg_duration_per_task_ms": 1005664.88, "avg_output_tokens_per_task": 8548.157142857142, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.7121428571428572, "accuracy_std_dev": 0.03208647153253476, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.13476882285714287, "avg_duration_per_task_ms": 139447.84285714285, "avg_output_tokens_per_task": 8900.078571428572, "last_run_timestamp": "2026-10-03T16:12:20.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.712121101010101, "accuracy_std_dev": 0.029111223892169177, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12218062626262625, "avg_duration_per_task_ms": 537818.2474747475, "avg_output_tokens_per_task": 55422.06060606061, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.71166675, "accuracy_std_dev": 0.019075733568790994, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07509325, "avg_duration_per_task_ms": 322731.32, "avg_output_tokens_per_task": 33483.435, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7115566504297995, "accuracy_std_dev": 0.03261413456694845, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.053420421203438394, "avg_duration_per_task_ms": 958453.7363896848, "avg_output_tokens_per_task": 13435.779369627508, "last_run_timestamp": "2026-10-04T00:00:34.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.7114285, "accuracy_std_dev": 0.029753450229103078, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.11265073714285717, "avg_duration_per_task_ms": 89786.91571428571, "avg_output_tokens_per_task": 10363.881428571429, "last_run_timestamp": "2026-10-03T16:12:29.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7113343768844221, "accuracy_std_dev": 0.031502220499778535, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.40403888609715255, "avg_duration_per_task_ms": 235745.01172529312, "avg_output_tokens_per_task": 9241.058626465661, "last_run_timestamp": "2026-10-03T08:37:32.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7111853238731218, "accuracy_std_dev": 0.034402559579759295, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.06973980801335558, "avg_duration_per_task_ms": 265076.1986644407, "avg_output_tokens_per_task": 8782.440734557596, "last_run_timestamp": "2026-10-03T16:18:22.330Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.011331263428624743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07186812, "avg_duration_per_task_ms": 348295.26, "avg_output_tokens_per_task": 33945.17333333333, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7109990505050505, "accuracy_std_dev": 0.0725087554956009, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.04814752861952862, "avg_duration_per_task_ms": 285404.8703703704, "avg_output_tokens_per_task": 27622.824915824916, "last_run_timestamp": "2026-10-03T16:03:45.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7103705555555555, "accuracy_std_dev": 0.030809822424065814, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03036840888888889, "avg_duration_per_task_ms": 105409.86666666667, "avg_output_tokens_per_task": 5076.06, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.7103704444444444, "accuracy_std_dev": 0.016662593743079532, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07196275999999999, "avg_duration_per_task_ms": 276773.88, "avg_output_tokens_per_task": 10708.504444444445, "last_run_timestamp": "2026-10-03T16:18:18.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12391315, "avg_duration_per_task_ms": 76197.77, "avg_output_tokens_per_task": 4134.025, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7099996666666667, "accuracy_std_dev": 0.03144660377352204, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.24028676333333332, "avg_duration_per_task_ms": 296408.56666666665, "avg_output_tokens_per_task": 20949.47, "last_run_timestamp": "2026-10-03T08:39:24.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.709697, "accuracy_std_dev": 0.02890091937197594, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.00848128909090909, "avg_duration_per_task_ms": 316978.2872727273, "avg_output_tokens_per_task": 14218.061818181819, "last_run_timestamp": "2026-10-03T08:22:39.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.709459418918919, "accuracy_std_dev": 0.03963897380384895, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.0811334054054054, "avg_duration_per_task_ms": 456032.14864864864, "avg_output_tokens_per_task": 13522.72972972973, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7091114702886249, "accuracy_std_dev": 0.03584624085968895, "run_count": 24, "total_questions": 1178, "avg_cost_per_task_usd": 0.09626076570458404, "avg_duration_per_task_ms": 76008.29456706283, "avg_output_tokens_per_task": 8032.944821731749, "last_run_timestamp": "2026-10-03T16:16:10.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7074906876404494, "accuracy_std_dev": 0.05861604916933776, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.05871908539325842, "avg_duration_per_task_ms": 920700.0044943821, "avg_output_tokens_per_task": 62280.74157303371, "last_run_timestamp": "2026-10-03T16:09:19.692Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08614948, "avg_duration_per_task_ms": 52133.28, "avg_output_tokens_per_task": 7649.3, "last_run_timestamp": "2026-07-18T08:04:46.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.22511376, "avg_duration_per_task_ms": 2937291.8, "avg_output_tokens_per_task": 55713.12, "last_run_timestamp": "2026-10-03T08:06:57.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7066666363636364, "accuracy_std_dev": 0.03527669560198404, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08133863090909091, "avg_duration_per_task_ms": 74950.49272727272, "avg_output_tokens_per_task": 8535.285454545454, "last_run_timestamp": "2026-10-03T16:09:05.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7063491754385965, "accuracy_std_dev": 0.03950815373122365, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.13163428320802006, "avg_duration_per_task_ms": 73963.29824561403, "avg_output_tokens_per_task": 10257.41604010025, "last_run_timestamp": "2026-08-29T17:11:32.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7056452258064517, "accuracy_std_dev": 0.012405393094940622, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.6654760282258064, "avg_duration_per_task_ms": 686538.5927419355, "avg_output_tokens_per_task": 55521.342741935485, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.7044674329896906, "accuracy_std_dev": 0.02710499999999999, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.11629732989690722, "avg_duration_per_task_ms": 255742.56701030929, "avg_output_tokens_per_task": 22402.164948453606, "last_run_timestamp": "2026-07-18T08:00:15.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.028782815862246673, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.059492744, "avg_duration_per_task_ms": 531413.344, "avg_output_tokens_per_task": 14170.264, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7039650806451613, "accuracy_std_dev": 0.09864976994620919, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.18442565725806453, "avg_duration_per_task_ms": 1083811.0504032257, "avg_output_tokens_per_task": 49448.979838709674, "last_run_timestamp": "2026-10-03T16:07:05.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7029631111111111, "accuracy_std_dev": 0.04275812214836289, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06788864444444445, "avg_duration_per_task_ms": 338428.77777777775, "avg_output_tokens_per_task": 10021.08, "last_run_timestamp": "2026-10-04T00:00:36.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7029628888888889, "accuracy_std_dev": 0.0630196289675675, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.2980291577777778, "avg_duration_per_task_ms": 402620.02222222224, "avg_output_tokens_per_task": 9612.011111111111, "last_run_timestamp": "2026-10-03T08:12:46.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.7028571428571428, "accuracy_std_dev": 0.027569581319317287, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.042805977142857145, "avg_duration_per_task_ms": 188835.22, "avg_output_tokens_per_task": 8445.88, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.70166675, "accuracy_std_dev": 0.13101104385198792, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0399951825, "avg_duration_per_task_ms": 280949.7025, "avg_output_tokens_per_task": 10610.9325, "last_run_timestamp": "2026-10-03T16:12:22.581Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.701111, "accuracy_std_dev": 0.011811220865487748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13736028333333333, "avg_duration_per_task_ms": 494538.36333333334, "avg_output_tokens_per_task": 57774.223333333335, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.7001918563218391, "accuracy_std_dev": 0.019518025856805777, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.01988737068965517, "avg_duration_per_task_ms": 465847.8132183908, "avg_output_tokens_per_task": 9664.399425287356, "last_run_timestamp": "2026-10-03T16:05:51.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.7000003333333334, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010915493333333333, "avg_duration_per_task_ms": 115952.82, "avg_output_tokens_per_task": 5345.16, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7000003333333332, "accuracy_std_dev": 0.024944382578492925, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12734470666666667, "avg_duration_per_task_ms": 591806.9066666666, "avg_output_tokens_per_task": 57532.753333333334, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.7000002, "accuracy_std_dev": 0.02828436552868031, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142434276, "avg_duration_per_task_ms": 459844.396, "avg_output_tokens_per_task": 11247.26, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.019795359999999998, "avg_duration_per_task_ms": 396615.94, "avg_output_tokens_per_task": 23218.96, "last_run_timestamp": "2026-10-01T08:03:06.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": 0.01440159456449179, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017548586666666668, "avg_duration_per_task_ms": 296799.9066666667, "avg_output_tokens_per_task": 10272.013333333334, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14202821999999998, "avg_duration_per_task_ms": 375738.88, "avg_output_tokens_per_task": 8552.48, "last_run_timestamp": "2026-07-11T15:34:16.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0112321, "avg_duration_per_task_ms": 596563.34, "avg_output_tokens_per_task": 4097.46, "last_run_timestamp": "2026-09-12T09:21:58.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.699275, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.02935723913043478, "avg_duration_per_task_ms": 277118.347826087, "avg_output_tokens_per_task": 12706.934782608696, "last_run_timestamp": "2026-10-03T16:14:03.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.6986666, "accuracy_std_dev": 0.007483243510671025, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04888268, "avg_duration_per_task_ms": 227205.572, "avg_output_tokens_per_task": 6367.684, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6983967174348699, "accuracy_std_dev": 0.025880910808740845, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.03737082565130261, "avg_duration_per_task_ms": 243092.8116232465, "avg_output_tokens_per_task": 9769.018036072144, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6977778333333333, "accuracy_std_dev": 0.017915990515148434, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21024033333333336, "avg_duration_per_task_ms": 335029.48333333334, "avg_output_tokens_per_task": 34166.46666666667, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.032659953959959374, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.025667106666666665, "avg_duration_per_task_ms": 221633.35777777777, "avg_output_tokens_per_task": 9824.533333333333, "last_run_timestamp": "2026-10-03T16:10:17.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6971726205357143, "accuracy_std_dev": 0.03692683540345123, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.045649031250000006, "avg_duration_per_task_ms": 440975.8839285714, "avg_output_tokens_per_task": 9165.5, "last_run_timestamp": "2026-10-03T16:26:46.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.6971425714285715, "accuracy_std_dev": 0.021412676644969954, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05162463142857143, "avg_duration_per_task_ms": 387031.3257142857, "avg_output_tokens_per_task": 9269.174285714285, "last_run_timestamp": "2026-10-03T16:12:07.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14710045000000002, "avg_duration_per_task_ms": 163276.98, "avg_output_tokens_per_task": 5053.91, "last_run_timestamp": "2026-10-01T08:02:59.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6951851111111111, "accuracy_std_dev": 0.07109563585519382, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05191746444444445, "avg_duration_per_task_ms": 304269.29333333333, "avg_output_tokens_per_task": 8106.5888888888885, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6936709012658228, "accuracy_std_dev": 0.02885537658495161, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.017572048101265822, "avg_duration_per_task_ms": 298381.053164557, "avg_output_tokens_per_task": 9343.311392405063, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.04422165130905898, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.029941567500000002, "avg_duration_per_task_ms": 308894.385, "avg_output_tokens_per_task": 10124.7675, "last_run_timestamp": "2026-10-03T16:07:02.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04339576, "avg_duration_per_task_ms": 106333.2, "avg_output_tokens_per_task": 10497.62, "last_run_timestamp": "2026-10-03T16:12:20.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020424833333333333, "avg_duration_per_task_ms": 454611.69333333336, "avg_output_tokens_per_task": 10363.686666666666, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01596628, "avg_duration_per_task_ms": 345497.04, "avg_output_tokens_per_task": 18466.26, "last_run_timestamp": "2026-10-03T16:11:33.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02187568, "avg_duration_per_task_ms": 236451.76, "avg_output_tokens_per_task": 11760.74, "last_run_timestamp": "2026-07-18T08:04:03.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.6933329999999999, "accuracy_std_dev": 0, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08326614666666667, "avg_duration_per_task_ms": 757569.06, "avg_output_tokens_per_task": 14889.873333333333, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6929165, "accuracy_std_dev": 0.02385937097609657, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.10850378125, "avg_duration_per_task_ms": 77457.025, "avg_output_tokens_per_task": 9370.97875, "last_run_timestamp": "2026-10-03T16:16:14.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.6925926777777779, "accuracy_std_dev": 0.05766701577287278, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.031123783333333335, "avg_duration_per_task_ms": 963670.3796296297, "avg_output_tokens_per_task": 42037.30925925926, "last_run_timestamp": "2026-10-03T16:10:32.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.692340005050505, "accuracy_std_dev": 0.01636465616379931, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.02100930303030303, "avg_duration_per_task_ms": 803131.053030303, "avg_output_tokens_per_task": 19211.46212121212, "last_run_timestamp": "2026-10-03T16:07:44.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.6920472487309645, "accuracy_std_dev": 0.10126504037171156, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.0741332766497462, "avg_duration_per_task_ms": 201116.1852791878, "avg_output_tokens_per_task": 25341.34771573604, "last_run_timestamp": "2026-10-03T16:12:22.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.68999975, "accuracy_std_dev": 0.03605553586742955, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.113432665, "avg_duration_per_task_ms": 675363.205, "avg_output_tokens_per_task": 27403.155, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6891892297297297, "accuracy_std_dev": 0.03170486152346707, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6622049662162162, "avg_duration_per_task_ms": 777791.8783783783, "avg_output_tokens_per_task": 59355.4527027027, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.6891667499999999, "accuracy_std_dev": 0.05343089124221587, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.029582090000000002, "avg_duration_per_task_ms": 175383.2625, "avg_output_tokens_per_task": 10636.405, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.6885714285714286, "accuracy_std_dev": 0.029913183528610335, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.19170214285714285, "avg_duration_per_task_ms": 1896431.7657142857, "avg_output_tokens_per_task": 13131.025714285714, "last_run_timestamp": "2026-10-04T00:00:37.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.688485, "accuracy_std_dev": 0.07169463814489388, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11140403636363637, "avg_duration_per_task_ms": 69132.01818181819, "avg_output_tokens_per_task": 8823.258181818182, "last_run_timestamp": "2026-10-03T16:27:27.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6872895959595959, "accuracy_std_dev": 0.028668953856916365, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.03334008585858586, "avg_duration_per_task_ms": 764612.6868686868, "avg_output_tokens_per_task": 19659.290404040403, "last_run_timestamp": "2026-10-03T16:26:51.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6866666666666666, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04308245333333333, "avg_duration_per_task_ms": 485038.98, "avg_output_tokens_per_task": 10019.313333333334, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.6866666, "accuracy_std_dev": 0.08753423203433042, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009446468, "avg_duration_per_task_ms": 231025.728, "avg_output_tokens_per_task": 4090.644, "last_run_timestamp": "2026-10-03T08:37:57.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.6858332500000001, "accuracy_std_dev": 0.009538074291359878, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.11306765, "avg_duration_per_task_ms": 138784.995, "avg_output_tokens_per_task": 8808.785, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.685, "accuracy_std_dev": 0.025110843165851677, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0074074375, "avg_duration_per_task_ms": 254406.3375, "avg_output_tokens_per_task": 11400.4, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6846240067340067, "accuracy_std_dev": 0.017472339200544633, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.01174172053872054, "avg_duration_per_task_ms": 245540.93602693602, "avg_output_tokens_per_task": 9163.602693602694, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.6844445555555555, "accuracy_std_dev": 0.052774778862868674, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09775986666666667, "avg_duration_per_task_ms": 262551.32, "avg_output_tokens_per_task": 10512.555555555555, "last_run_timestamp": "2026-10-03T16:11:17.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6841667499999999, "accuracy_std_dev": 0.042023533421018985, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.03507941999999999, "avg_duration_per_task_ms": 165936.86375, "avg_output_tokens_per_task": 10189.80625, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6838585558739255, "accuracy_std_dev": 0.02128582066608127, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.006118083094555874, "avg_duration_per_task_ms": 239133.92550143268, "avg_output_tokens_per_task": 9429.15759312321, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.68333325, "accuracy_std_dev": 0.06368309125024867, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.05480554, "avg_duration_per_task_ms": 1657352.435, "avg_output_tokens_per_task": 54132.445, "last_run_timestamp": "2026-10-03T08:39:18.140Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.683333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05961793, "avg_duration_per_task_ms": 370273.51, "avg_output_tokens_per_task": 10007.47, "last_run_timestamp": "2026-07-18T08:04:08.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6820807398843931, "accuracy_std_dev": 0.018427811270036426, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.015085965317919076, "avg_duration_per_task_ms": 243924.5809248555, "avg_output_tokens_per_task": 11146.326589595375, "last_run_timestamp": "2026-08-30T00:01:09.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6816665000000001, "accuracy_std_dev": 0.02327362599059285, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013877145, "avg_duration_per_task_ms": 70751.9, "avg_output_tokens_per_task": 5304.49, "last_run_timestamp": "2026-10-01T19:04:52.652Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6791665, "accuracy_std_dev": 0.04377175966876361, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.020768927500000003, "avg_duration_per_task_ms": 425360.89, "avg_output_tokens_per_task": 20325.145, "last_run_timestamp": "2026-10-03T16:11:33.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6785186666666666, "accuracy_std_dev": 0.019570227381861923, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.00874342888888889, "avg_duration_per_task_ms": 244185.2288888889, "avg_output_tokens_per_task": 10575.206666666667, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.677500125, "accuracy_std_dev": 0.09463012829225889, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08997287749999999, "avg_duration_per_task_ms": 87500.14, "avg_output_tokens_per_task": 8449.92, "last_run_timestamp": "2026-10-03T16:14:25.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.676667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0194754, "avg_duration_per_task_ms": 233472.04, "avg_output_tokens_per_task": 10655.96, "last_run_timestamp": "2026-07-11T15:34:15.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09087626, "avg_duration_per_task_ms": 1373764.79, "avg_output_tokens_per_task": 16227.07, "last_run_timestamp": "2026-07-18T16:00:08.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64995528, "avg_duration_per_task_ms": 85231.23, "avg_output_tokens_per_task": 3880.06, "last_run_timestamp": "2026-10-01T08:05:26.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06914647, "avg_duration_per_task_ms": 138661.04, "avg_output_tokens_per_task": 2568.53, "last_run_timestamp": "2026-10-01T16:01:04.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.02372680934208296, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11501549333333332, "avg_duration_per_task_ms": 145439.36666666667, "avg_output_tokens_per_task": 56059.84, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.05192585882668568, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.22792725999999996, "avg_duration_per_task_ms": 692249.9666666667, "avg_output_tokens_per_task": 10865.973333333333, "last_run_timestamp": "2026-10-03T16:12:13.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6715151818181818, "accuracy_std_dev": 0.0638870956593358, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06997897636363637, "avg_duration_per_task_ms": 726161.1018181818, "avg_output_tokens_per_task": 18626.429090909092, "last_run_timestamp": "2026-10-03T08:12:20.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.671111, "accuracy_std_dev": 0.034282673881196074, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1170691888888889, "avg_duration_per_task_ms": 122970.47333333333, "avg_output_tokens_per_task": 12647.033333333333, "last_run_timestamp": "2026-10-03T08:12:32.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.67083325, "accuracy_std_dev": 0.019346895388860186, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.017078424999999998, "avg_duration_per_task_ms": 126227.985, "avg_output_tokens_per_task": 21265.575, "last_run_timestamp": "2026-10-03T08:15:42.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.67, "accuracy_std_dev": 0.037267605832679934, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02244593, "avg_duration_per_task_ms": 202743.04, "avg_output_tokens_per_task": 31421.765, "last_run_timestamp": "2026-10-03T08:12:46.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.67, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014700359999999999, "avg_duration_per_task_ms": 1773933.08, "avg_output_tokens_per_task": 23989.78, "last_run_timestamp": "2026-07-18T08:04:08.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6692307307692308, "accuracy_std_dev": 0.031054260799641166, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.12451571153846153, "avg_duration_per_task_ms": 119043.02846153846, "avg_output_tokens_per_task": 12553.901538461538, "last_run_timestamp": "2026-10-03T08:32:52.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.669204852791878, "accuracy_std_dev": 0.0421484109961455, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.023811266497461927, "avg_duration_per_task_ms": 266573.3654822335, "avg_output_tokens_per_task": 17577.88578680203, "last_run_timestamp": "2026-10-03T16:10:49.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6690613965517241, "accuracy_std_dev": 0.061175733864975976, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.023726439655172418, "avg_duration_per_task_ms": 462159.5632183908, "avg_output_tokens_per_task": 16750.870689655174, "last_run_timestamp": "2026-10-03T16:16:54.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.66833325, "accuracy_std_dev": 0.023746444179866155, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.016393715, "avg_duration_per_task_ms": 283331.04, "avg_output_tokens_per_task": 10626.035, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03810232, "avg_duration_per_task_ms": 661152.42, "avg_output_tokens_per_task": 27538.94, "last_run_timestamp": "2026-07-11T15:34:14.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6658330000000001, "accuracy_std_dev": 0.014790199457749011, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02195716, "avg_duration_per_task_ms": 162018.975, "avg_output_tokens_per_task": 16412.775, "last_run_timestamp": "2026-10-03T16:06:47.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.663333375, "accuracy_std_dev": 0.022360549338832768, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.029954157500000002, "avg_duration_per_task_ms": 239902.565, "avg_output_tokens_per_task": 11150.49, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6603704444444444, "accuracy_std_dev": 0.02063470181843689, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11393844222222221, "avg_duration_per_task_ms": 124032.99777777778, "avg_output_tokens_per_task": 12195, "last_run_timestamp": "2026-10-03T08:07:11.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.66, "accuracy_std_dev": 0.04000011666732984, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.018718389999999998, "avg_duration_per_task_ms": 348125.166, "avg_output_tokens_per_task": 19960.056, "last_run_timestamp": "2026-10-03T16:07:05.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.66, "accuracy_std_dev": 0.055243555012729276, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.022544044444444442, "avg_duration_per_task_ms": 206377.97777777776, "avg_output_tokens_per_task": 9087.275555555556, "last_run_timestamp": "2026-10-03T16:13:50.327Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00861218, "avg_duration_per_task_ms": 379136.4, "avg_output_tokens_per_task": 24201, "last_run_timestamp": "2026-07-18T16:00:10.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.66, "accuracy_std_dev": 0.08092060123218178, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07575899333333333, "avg_duration_per_task_ms": 264060.94444444444, "avg_output_tokens_per_task": 24785.226666666666, "last_run_timestamp": "2026-10-03T16:03:45.631Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.659524, "accuracy_std_dev": 0.0937819106087553, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.058310568571428564, "avg_duration_per_task_ms": 524817.9057142857, "avg_output_tokens_per_task": 17039.157142857144, "last_run_timestamp": "2026-10-03T16:12:24.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.6580951428571429, "accuracy_std_dev": 0.021884097914947755, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.022360888571428574, "avg_duration_per_task_ms": 259146.80571428573, "avg_output_tokens_per_task": 9927.357142857143, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.65666675, "accuracy_std_dev": 0.024720818937436124, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02259439, "avg_duration_per_task_ms": 296251.1175, "avg_output_tokens_per_task": 10399.475, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6566666250000001, "accuracy_std_dev": 0.017638208213545233, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.026674842500000004, "avg_duration_per_task_ms": 368698.9125, "avg_output_tokens_per_task": 10712.57, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.6566665, "accuracy_std_dev": 0.0833335, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.009373, "avg_duration_per_task_ms": 175154.56, "avg_output_tokens_per_task": 15943.33, "last_run_timestamp": "2026-10-01T16:01:04.874Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6566665, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.008972560000000001, "avg_duration_per_task_ms": 569999.06, "avg_output_tokens_per_task": 21009.53, "last_run_timestamp": "2026-10-03T16:14:02.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6560493333333334, "accuracy_std_dev": 0.029495577726286477, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.12169699629629627, "avg_duration_per_task_ms": 120224.82148148149, "avg_output_tokens_per_task": 12512.395555555555, "last_run_timestamp": "2026-10-03T08:13:54.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6557788844221106, "accuracy_std_dev": 0.01812503680513505, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.017334296482412057, "avg_duration_per_task_ms": 354124.29145728645, "avg_output_tokens_per_task": 22535.824120603014, "last_run_timestamp": "2026-10-03T08:21:59.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6555554666666666, "accuracy_std_dev": 0.010407070153866542, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.014745425641025642, "avg_duration_per_task_ms": 278015.8512820513, "avg_output_tokens_per_task": 20205.384615384617, "last_run_timestamp": "2026-10-03T08:06:43.564Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.023726809342082957, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03210012, "avg_duration_per_task_ms": 143919.89333333334, "avg_output_tokens_per_task": 17353.273333333334, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.65166675, "accuracy_std_dev": 0.012801703448662586, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.031589305, "avg_duration_per_task_ms": 139715.76, "avg_output_tokens_per_task": 17049.6, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.6503832528735632, "accuracy_std_dev": 0.09824690755999416, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07506860057471264, "avg_duration_per_task_ms": 844732.393678161, "avg_output_tokens_per_task": 47265.1724137931, "last_run_timestamp": "2026-10-03T16:13:53.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.649166625, "accuracy_std_dev": 0.029238002051856674, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021406732500000004, "avg_duration_per_task_ms": 233528.235, "avg_output_tokens_per_task": 10579.235, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.6485233670886075, "accuracy_std_dev": 0.02680153729616082, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.011709784810126583, "avg_duration_per_task_ms": 286395.1898734177, "avg_output_tokens_per_task": 16354.617721518987, "last_run_timestamp": "2026-10-03T16:05:34.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6483331666666667, "accuracy_std_dev": 0.0255133770299469, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022298810000000002, "avg_duration_per_task_ms": 289277.4166666667, "avg_output_tokens_per_task": 10734.94, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6481482222222222, "accuracy_std_dev": 0.02470134810436142, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11660464888888887, "avg_duration_per_task_ms": 145744.04888888888, "avg_output_tokens_per_task": 12598.522222222222, "last_run_timestamp": "2026-10-03T08:13:37.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.6481482020202021, "accuracy_std_dev": 0.0054059166891471805, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.009114631313131312, "avg_duration_per_task_ms": 604721.6666666666, "avg_output_tokens_per_task": 19746.29292929293, "last_run_timestamp": "2026-08-22T16:10:30.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6477778333333334, "accuracy_std_dev": 0.03430091421724631, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09916395333333333, "avg_duration_per_task_ms": 153090.92333333334, "avg_output_tokens_per_task": 12859.136666666667, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.6465774285714286, "accuracy_std_dev": 0.10466666390033022, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.09237277678571428, "avg_duration_per_task_ms": 238254.9263392857, "avg_output_tokens_per_task": 25810.723214285714, "last_run_timestamp": "2026-10-03T16:12:24.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.646566095477387, "accuracy_std_dev": 0.12986162562313927, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.04628975628140704, "avg_duration_per_task_ms": 2127959.7989949747, "avg_output_tokens_per_task": 71164.3944723618, "last_run_timestamp": "2026-10-03T08:42:41.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.6454141073825505, "accuracy_std_dev": 0.01972979797722779, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.018687657718120806, "avg_duration_per_task_ms": 171442.07382550335, "avg_output_tokens_per_task": 9143.295302013423, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.644166625, "accuracy_std_dev": 0.02184215629772792, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.024566542500000007, "avg_duration_per_task_ms": 157744.9625, "avg_output_tokens_per_task": 10696.215, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6433335, "accuracy_std_dev": 0.014272415770172448, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031060726666666667, "avg_duration_per_task_ms": 229987.72666666665, "avg_output_tokens_per_task": 16358.803333333333, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6433333333333333, "accuracy_std_dev": 0.14455548467015086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10531282, "avg_duration_per_task_ms": 1453976.54, "avg_output_tokens_per_task": 18878.353333333333, "last_run_timestamp": "2026-08-08T08:21:31.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.64333325, "accuracy_std_dev": 0.024267181772663682, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024900534999999998, "avg_duration_per_task_ms": 168518.28, "avg_output_tokens_per_task": 11257.595, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6415409547738694, "accuracy_std_dev": 0.019239053075710364, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.012467824120603016, "avg_duration_per_task_ms": 963027.175879397, "avg_output_tokens_per_task": 22096.447236180906, "last_run_timestamp": "2026-10-03T16:01:59.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6411111111111111, "accuracy_std_dev": 0.028458355974395857, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.032843908888888884, "avg_duration_per_task_ms": 84121.66888888889, "avg_output_tokens_per_task": 20372.67111111111, "last_run_timestamp": "2026-10-03T16:20:43.625Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.64, "accuracy_std_dev": 0.025647213408087847, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.218415728, "avg_duration_per_task_ms": 1219489.072, "avg_output_tokens_per_task": 31443.888, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.63916675, "accuracy_std_dev": 0.03546318317900269, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.014215240000000002, "avg_duration_per_task_ms": 1323984.41, "avg_output_tokens_per_task": 23296.155, "last_run_timestamp": "2026-10-03T08:02:52.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.04228062043326973, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08944130666666666, "avg_duration_per_task_ms": 118157.82666666666, "avg_output_tokens_per_task": 12612.513333333334, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.63416675, "accuracy_std_dev": 0.030630012062314006, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01972235, "avg_duration_per_task_ms": 313247.5575, "avg_output_tokens_per_task": 21107.5425, "last_run_timestamp": "2026-10-03T16:05:02.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.6336735020408163, "accuracy_std_dev": 0.10052381662193295, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.02281844081632653, "avg_duration_per_task_ms": 906837.293877551, "avg_output_tokens_per_task": 30461.944897959183, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6314878027681661, "accuracy_std_dev": 0.07985108320370836, "run_count": 6, "total_questions": 289, "avg_cost_per_task_usd": 0.14447499307958475, "avg_duration_per_task_ms": 358430.0657439446, "avg_output_tokens_per_task": 10974.453287197231, "last_run_timestamp": "2026-08-29T16:44:17.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.63, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07672618, "avg_duration_per_task_ms": 231628.06, "avg_output_tokens_per_task": 10580.58, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.63, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07387748, "avg_duration_per_task_ms": 1456099.47, "avg_output_tokens_per_task": 24632.34, "last_run_timestamp": "2026-07-18T16:00:08.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.627499875, "accuracy_std_dev": 0.020121193568210006, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.028656730000000002, "avg_duration_per_task_ms": 222093.8825, "avg_output_tokens_per_task": 10287.135, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6256345228426395, "accuracy_std_dev": 0.03217519695525384, "run_count": 16, "total_questions": 788, "avg_cost_per_task_usd": 0.020946473350253805, "avg_duration_per_task_ms": 314363.8718274112, "avg_output_tokens_per_task": 19844.57614213198, "last_run_timestamp": "2026-10-03T16:12:29.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6247562046783627, "accuracy_std_dev": 0.10453632978607418, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.43554279824561404, "avg_duration_per_task_ms": 2351365.956140351, "avg_output_tokens_per_task": 74116.19883040935, "last_run_timestamp": "2026-10-03T08:06:05.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.6245014239316239, "accuracy_std_dev": 0.06171917805736228, "run_count": 12, "total_questions": 585, "avg_cost_per_task_usd": 0.022623485470085473, "avg_duration_per_task_ms": 249317.92307692306, "avg_output_tokens_per_task": 24519.533333333333, "last_run_timestamp": "2026-10-03T16:12:24.693Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.624, "accuracy_std_dev": 0.012365076271499492, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.11120451600000002, "avg_duration_per_task_ms": 725043.5, "avg_output_tokens_per_task": 32381.232, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6216667499999999, "accuracy_std_dev": 0.04997217837144482, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08547625, "avg_duration_per_task_ms": 106037.93, "avg_output_tokens_per_task": 11996.815, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.6211111666666667, "accuracy_std_dev": 0.18460855717654465, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13787135666666667, "avg_duration_per_task_ms": 390996.2266666667, "avg_output_tokens_per_task": 8010.466666666666, "last_run_timestamp": "2026-10-03T16:12:26.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6211110000000001, "accuracy_std_dev": 0.041171085383636265, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09511168999999998, "avg_duration_per_task_ms": 55650.66, "avg_output_tokens_per_task": 3586.0333333333333, "last_run_timestamp": "2026-10-03T16:14:18.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.6208576842105263, "accuracy_std_dev": 0.07290857624841833, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.08742642982456139, "avg_duration_per_task_ms": 1045323.0116959064, "avg_output_tokens_per_task": 62138.06140350877, "last_run_timestamp": "2026-10-03T16:14:02.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.6199997500000001, "accuracy_std_dev": 0.019999916667013916, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02507946, "avg_duration_per_task_ms": 173259.145, "avg_output_tokens_per_task": 11472.255, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6187741034482758, "accuracy_std_dev": 0.01093895001490416, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.11070118965517242, "avg_duration_per_task_ms": 713990.3045977012, "avg_output_tokens_per_task": 33324.629310344826, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6180555, "accuracy_std_dev": 0.00694450000000002, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.025814302083333334, "avg_duration_per_task_ms": 76875.0625, "avg_output_tokens_per_task": 17301.635416666668, "last_run_timestamp": "2026-08-29T16:33:38.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.6166667499999999, "accuracy_std_dev": 0.06236093417507071, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.037265545000000004, "avg_duration_per_task_ms": 382273.955, "avg_output_tokens_per_task": 20068.4325, "last_run_timestamp": "2026-10-03T16:12:10.894Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6158248141414141, "accuracy_std_dev": 0.2285695922821756, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.020359361616161618, "avg_duration_per_task_ms": 340617.86262626265, "avg_output_tokens_per_task": 10409.925252525252, "last_run_timestamp": "2026-09-26T08:14:40.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.61499975, "accuracy_std_dev": 0.03662873416714669, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.091530915, "avg_duration_per_task_ms": 150799.645, "avg_output_tokens_per_task": 3905.64, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.61458325, "accuracy_std_dev": 0.028281053484400142, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.065036325, "avg_duration_per_task_ms": 29159.9975, "avg_output_tokens_per_task": 20314.995, "last_run_timestamp": "2026-10-03T16:18:39.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6133331666666667, "accuracy_std_dev": 0.031972140650972335, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04558941666666667, "avg_duration_per_task_ms": 201450.47, "avg_output_tokens_per_task": 3388.8166666666666, "last_run_timestamp": "2026-10-01T16:01:02.677Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.613333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01616616, "avg_duration_per_task_ms": 108573.58, "avg_output_tokens_per_task": 21305.1, "last_run_timestamp": "2026-10-03T16:13:54.745Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.608, "accuracy_std_dev": 0.11488351975457575, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.058990216, "avg_duration_per_task_ms": 582295.388, "avg_output_tokens_per_task": 35784.112, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6070889547511311, "accuracy_std_dev": 0.0286460576026238, "run_count": 9, "total_questions": 442, "avg_cost_per_task_usd": 0.041861986425339365, "avg_duration_per_task_ms": 87046.60407239819, "avg_output_tokens_per_task": 24951.95701357466, "last_run_timestamp": "2026-10-03T16:12:29.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.6066666666666667, "accuracy_std_dev": 0.15241273398753646, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017056396666666668, "avg_duration_per_task_ms": 233868.72666666665, "avg_output_tokens_per_task": 8308.71, "last_run_timestamp": "2026-08-29T16:33:39.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6060269955357143, "accuracy_std_dev": 0.032348691330028045, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.19772456473214287, "avg_duration_per_task_ms": 188670.52455357142, "avg_output_tokens_per_task": 13047.183035714286, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.603333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.030717460000000002, "avg_duration_per_task_ms": 422658.18, "avg_output_tokens_per_task": 11680.98, "last_run_timestamp": "2026-07-18T16:00:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.6029999, "accuracy_std_dev": 0.026010556404852253, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04766987600000001, "avg_duration_per_task_ms": 92389.166, "avg_output_tokens_per_task": 4752.396, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.6019345223214286, "accuracy_std_dev": 0.023580042960096588, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.019949879464285714, "avg_duration_per_task_ms": 99253.61830357143, "avg_output_tokens_per_task": 4312.604910714285, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.599552610738255, "accuracy_std_dev": 0.021214776019452947, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.1599610402684564, "avg_duration_per_task_ms": 609782.6946308725, "avg_output_tokens_per_task": 29203.84899328859, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.5933333333333334, "accuracy_std_dev": 0.02880349773127486, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21649743333333332, "avg_duration_per_task_ms": 191120.76666666666, "avg_output_tokens_per_task": 14122.313333333334, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.5909091212121212, "accuracy_std_dev": 0.04025476018668477, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.0993339494949495, "avg_duration_per_task_ms": 115416.30808080808, "avg_output_tokens_per_task": 3700.722222222222, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5881481111111111, "accuracy_std_dev": 0.007554077566224961, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029889999999999996, "avg_duration_per_task_ms": 78610.16, "avg_output_tokens_per_task": 8108.122222222222, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5799998333333334, "accuracy_std_dev": 0.042339037236797064, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01084836, "avg_duration_per_task_ms": 298654.7033333333, "avg_output_tokens_per_task": 18007.093333333334, "last_run_timestamp": "2026-08-29T16:33:38.124Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5791477009132421, "accuracy_std_dev": 0.037940164496542333, "run_count": 9, "total_questions": 438, "avg_cost_per_task_usd": 0.07539671461187215, "avg_duration_per_task_ms": 91646.01598173517, "avg_output_tokens_per_task": 5547.684931506849, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.571111, "accuracy_std_dev": 0.008314764498569157, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028662713333333336, "avg_duration_per_task_ms": 78035.85333333333, "avg_output_tokens_per_task": 8001.593333333333, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.5622223333333333, "accuracy_std_dev": 0.03456997912768952, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.23936231333333335, "avg_duration_per_task_ms": 165980.19333333333, "avg_output_tokens_per_task": 13303.673333333334, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5616569826589596, "accuracy_std_dev": 0.040448501937396776, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.177089710982659, "avg_duration_per_task_ms": 1805453.9132947978, "avg_output_tokens_per_task": 38595.37283236994, "last_run_timestamp": "2026-10-03T16:17:09.625Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5604491347150259, "accuracy_std_dev": 0.10678459157041047, "run_count": 8, "total_questions": 386, "avg_cost_per_task_usd": 0.07120025388601035, "avg_duration_per_task_ms": 510444.22279792745, "avg_output_tokens_per_task": 29300.067357512955, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.5572390808080808, "accuracy_std_dev": 0.08974967405616299, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04370982828282828, "avg_duration_per_task_ms": 3081206.1666666665, "avg_output_tokens_per_task": 93737.79292929293, "last_run_timestamp": "2026-10-03T16:33:53.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5521540748299321, "accuracy_std_dev": 0.03338584001572458, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.0026003809523809525, "avg_duration_per_task_ms": 390785.49659863947, "avg_output_tokens_per_task": 6412.292517006803, "last_run_timestamp": "2026-10-03T16:14:16.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.5516073604060914, "accuracy_std_dev": 0.07298347728499496, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.051452035532994926, "avg_duration_per_task_ms": 435341.5304568528, "avg_output_tokens_per_task": 9213.700507614214, "last_run_timestamp": "2026-10-04T00:00:28.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5511113333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.035753513333333334, "avg_duration_per_task_ms": 61316.40666666667, "avg_output_tokens_per_task": 6659.566666666667, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.5496295555555556, "accuracy_std_dev": 0.09428684395928454, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04100789111111111, "avg_duration_per_task_ms": 99006.21111111112, "avg_output_tokens_per_task": 4319.34, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.5375, "accuracy_std_dev": 0.09219167231642998, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.029294462499999997, "avg_duration_per_task_ms": 409054.695, "avg_output_tokens_per_task": 11072.1825, "last_run_timestamp": "2026-10-03T16:13:14.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5355556666666667, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09782582666666667, "avg_duration_per_task_ms": 375218.6066666667, "avg_output_tokens_per_task": 16827.4, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.533914726744186, "accuracy_std_dev": 0.015990278830002442, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.02433947674418605, "avg_duration_per_task_ms": 1793934.9767441861, "avg_output_tokens_per_task": 124117.57848837209, "last_run_timestamp": "2026-10-03T16:10:46.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.526667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03416076, "avg_duration_per_task_ms": 1845834.04, "avg_output_tokens_per_task": 105339.92, "last_run_timestamp": "2026-07-18T16:00:08.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.523035243902439, "accuracy_std_dev": 0.26305710961614404, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.01812669918699187, "avg_duration_per_task_ms": 110026.72764227643, "avg_output_tokens_per_task": 18887.439024390245, "last_run_timestamp": "2026-08-29T17:06:47.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5221089591836734, "accuracy_std_dev": 0.023889008586586408, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.13644156632653062, "avg_duration_per_task_ms": 73125.92346938775, "avg_output_tokens_per_task": 2553.7397959183672, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517500125, "accuracy_std_dev": 0.024930426803594342, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0282700725, "avg_duration_per_task_ms": 444815, "avg_output_tokens_per_task": 58343.125, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5156599127516779, "accuracy_std_dev": 0.03449671802746843, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.024214489932885906, "avg_duration_per_task_ms": 190286.47651006712, "avg_output_tokens_per_task": 48937.10738255033, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.5125241502890173, "accuracy_std_dev": 0.04228836158315612, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.14881240173410404, "avg_duration_per_task_ms": 1730003.4537572255, "avg_output_tokens_per_task": 33598.51734104046, "last_run_timestamp": "2026-10-03T16:05:51.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5119049598214286, "accuracy_std_dev": 0.02807387800381059, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.06533608482142857, "avg_duration_per_task_ms": 299963.359375, "avg_output_tokens_per_task": 4007.223214285714, "last_run_timestamp": "2026-10-04T00:00:34.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.5104164999999999, "accuracy_std_dev": 0.003472500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.024820281250000003, "avg_duration_per_task_ms": 480070.4583333333, "avg_output_tokens_per_task": 9578.8125, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.51, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.022937250000000003, "avg_duration_per_task_ms": 2318885.96, "avg_output_tokens_per_task": 152067.3, "last_run_timestamp": "2026-07-18T16:00:10.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5083332500000001, "accuracy_std_dev": 0.026404802805313633, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12635733, "avg_duration_per_task_ms": 53878.245, "avg_output_tokens_per_task": 2800.47, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5033784729729729, "accuracy_std_dev": 0.012185696720153325, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.021273824324324325, "avg_duration_per_task_ms": 1274174.0878378379, "avg_output_tokens_per_task": 117319.30743243243, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5033670303030303, "accuracy_std_dev": 0.030340499999999992, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.01769851515151515, "avg_duration_per_task_ms": 135563.53535353535, "avg_output_tokens_per_task": 30988.575757575756, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.5033669747474748, "accuracy_std_dev": 0.027292446564651823, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.025276146464646466, "avg_duration_per_task_ms": 577739.3712121212, "avg_output_tokens_per_task": 53598.469696969696, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.49740733333333337, "accuracy_std_dev": 0.030943206527802244, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02629459333333333, "avg_duration_per_task_ms": 283072.18, "avg_output_tokens_per_task": 19100.475555555557, "last_run_timestamp": "2026-10-03T16:15:51.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23109727999999996, "avg_duration_per_task_ms": 80168.37, "avg_output_tokens_per_task": 2746.03, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.49523814285714285, "accuracy_std_dev": 0.023289352137395136, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06688620571428572, "avg_duration_per_task_ms": 308997.38285714283, "avg_output_tokens_per_task": 4074.4114285714286, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.49319713265306125, "accuracy_std_dev": 0.03082350135347216, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.05686764285714286, "avg_duration_per_task_ms": 2675498.8775510206, "avg_output_tokens_per_task": 42954.67857142857, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.489899005050505, "accuracy_std_dev": 0.009452416119701875, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.014275666666666666, "avg_duration_per_task_ms": 117584.4797979798, "avg_output_tokens_per_task": 3141.5757575757575, "last_run_timestamp": "2026-10-03T16:05:51.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.4894028092485549, "accuracy_std_dev": 0.025397205898062335, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.023809176300578036, "avg_duration_per_task_ms": 845136.3179190751, "avg_output_tokens_per_task": 22279.682080924857, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48833333333333334, "accuracy_std_dev": 0.018831396165859726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03323320666666667, "avg_duration_per_task_ms": 665027.6466666666, "avg_output_tokens_per_task": 61694.416666666664, "last_run_timestamp": "2026-10-03T16:04:32.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.4852150080645161, "accuracy_std_dev": 0.023420064714684302, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.028619552419354842, "avg_duration_per_task_ms": 386610.2439516129, "avg_output_tokens_per_task": 66355.3125, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.4825, "accuracy_std_dev": 0.02355551362738668, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0347383225, "avg_duration_per_task_ms": 43524, "avg_output_tokens_per_task": 3653.295, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.47833325000000004, "accuracy_std_dev": 0.028234971421758143, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.022111440000000003, "avg_duration_per_task_ms": 722757.885, "avg_output_tokens_per_task": 21962.2775, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.4777776666666667, "accuracy_std_dev": 0.013698571612479242, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12510532, "avg_duration_per_task_ms": 53067.566666666666, "avg_output_tokens_per_task": 2709.2933333333335, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47701135057471267, "accuracy_std_dev": 0.009988678401274195, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.16442305459770115, "avg_duration_per_task_ms": 2471430.7183908047, "avg_output_tokens_per_task": 44221.49137931035, "last_run_timestamp": "2026-10-03T16:12:22.579Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.4762964444444444, "accuracy_std_dev": 0.03444249058651928, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.023876493333333335, "avg_duration_per_task_ms": 58280.74444444444, "avg_output_tokens_per_task": 3755.9466666666667, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.475168393939394, "accuracy_std_dev": 0.08821191621175678, "run_count": 16, "total_questions": 792, "avg_cost_per_task_usd": 0.04045829419191919, "avg_duration_per_task_ms": 704580.8383838384, "avg_output_tokens_per_task": 194395.8547979798, "last_run_timestamp": "2026-10-03T16:19:24.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.473333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03316041, "avg_duration_per_task_ms": 478208.83, "avg_output_tokens_per_task": 36953.92, "last_run_timestamp": "2026-07-18T16:00:08.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.4727888571428572, "accuracy_std_dev": 0.053216013468691904, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.026510076530612242, "avg_duration_per_task_ms": 2225913.1275510206, "avg_output_tokens_per_task": 135590.98469387754, "last_run_timestamp": "2026-10-03T16:09:05.687Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.47083349999999996, "accuracy_std_dev": 0.03247871952910089, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0190439575, "avg_duration_per_task_ms": 88315.37, "avg_output_tokens_per_task": 3656.9575, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4696295555555556, "accuracy_std_dev": 0.026314631063814885, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024747560000000002, "avg_duration_per_task_ms": 125553.2, "avg_output_tokens_per_task": 3799.4177777777777, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4682815259515572, "accuracy_std_dev": 0.039807100596362034, "run_count": 6, "total_questions": 289, "avg_cost_per_task_usd": 0.04939322491349481, "avg_duration_per_task_ms": 229004.3321799308, "avg_output_tokens_per_task": 98239.9169550173, "last_run_timestamp": "2026-10-03T16:14:31.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.4666665555555556, "accuracy_std_dev": 0.031426983767001204, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10525973333333334, "avg_duration_per_task_ms": 99178.76444444444, "avg_output_tokens_per_task": 2760.7244444444445, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.4662164729729729, "accuracy_std_dev": 0.03892660093903227, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04219703378378378, "avg_duration_per_task_ms": 711257.6756756756, "avg_output_tokens_per_task": 98782.22297297297, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4653333, "accuracy_std_dev": 0.0479628643390905, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.054455583999999994, "avg_duration_per_task_ms": 126748.042, "avg_output_tokens_per_task": 3614.196, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.4652775, "accuracy_std_dev": 0.006944499999999992, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.027522666666666668, "avg_duration_per_task_ms": 29612.604166666668, "avg_output_tokens_per_task": 1399.6145833333333, "last_run_timestamp": "2026-07-18T16:00:08.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4640741111111111, "accuracy_std_dev": 0.03249991284242824, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02241910888888889, "avg_duration_per_task_ms": 213262.18444444446, "avg_output_tokens_per_task": 3634.297777777778, "last_run_timestamp": "2026-10-03T16:04:32.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.4626308744394619, "accuracy_std_dev": 0.044603677582420745, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.054874065022421516, "avg_duration_per_task_ms": 71821.13004484306, "avg_output_tokens_per_task": 3451.358744394619, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4608501677852349, "accuracy_std_dev": 0.034994868451198195, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04671855033557047, "avg_duration_per_task_ms": 767600.1241610738, "avg_output_tokens_per_task": 101728.0033557047, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.46068351282051284, "accuracy_std_dev": 0.04225414673955088, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.01900871794871795, "avg_duration_per_task_ms": 170098.1794871795, "avg_output_tokens_per_task": 35854.89743589744, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.45962955555555557, "accuracy_std_dev": 0.03844019636934208, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.053124257777777774, "avg_duration_per_task_ms": 153327.5977777778, "avg_output_tokens_per_task": 3785.157777777778, "last_run_timestamp": "2026-10-03T16:12:33.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.45920115624999996, "accuracy_std_dev": 0.020597873615982785, "run_count": 4, "total_questions": 192, "avg_cost_per_task_usd": 0.08208188020833333, "avg_duration_per_task_ms": 47993.442708333336, "avg_output_tokens_per_task": 3348.3072916666665, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.45542626162790695, "accuracy_std_dev": 0.018912652487751393, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.046210372093023255, "avg_duration_per_task_ms": 174189.27034883722, "avg_output_tokens_per_task": 1898.7906976744187, "last_run_timestamp": "2026-08-29T16:23:01.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.45333349999999994, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.2293187, "avg_duration_per_task_ms": 78949.14, "avg_output_tokens_per_task": 2726.37, "last_run_timestamp": "2026-07-18T08:04:08.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.451852, "accuracy_std_dev": 0.06252139759936415, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.15496766666666664, "avg_duration_per_task_ms": 120175.53777777778, "avg_output_tokens_per_task": 3250.7844444444445, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4511785656565656, "accuracy_std_dev": 0.03515039216708685, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04521708585858586, "avg_duration_per_task_ms": 779859.6717171717, "avg_output_tokens_per_task": 105433.07575757576, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.44962944444444447, "accuracy_std_dev": 0.027054818100832535, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.032987291111111115, "avg_duration_per_task_ms": 107424.01111111112, "avg_output_tokens_per_task": 3035.311111111111, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.44925922222222225, "accuracy_std_dev": 0.030622329285008784, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04105088444444445, "avg_duration_per_task_ms": 296366.4222222222, "avg_output_tokens_per_task": 3639.817777777778, "last_run_timestamp": "2026-10-03T16:17:30.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.44857142857142857, "accuracy_std_dev": 0.025125762588894082, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021147157142857143, "avg_duration_per_task_ms": 505078.46285714285, "avg_output_tokens_per_task": 21834.525714285715, "last_run_timestamp": "2026-08-22T16:08:38.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.44388611055276384, "accuracy_std_dev": 0.025511183797197155, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.01678875879396985, "avg_duration_per_task_ms": 204045.45477386934, "avg_output_tokens_per_task": 2857.608040201005, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.44333333333333336, "accuracy_std_dev": 0.025962822424039775, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028878013333333334, "avg_duration_per_task_ms": 681765.0866666667, "avg_output_tokens_per_task": 19566.68, "last_run_timestamp": "2026-10-01T16:01:00.465Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.44333325, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025700495, "avg_duration_per_task_ms": 61023.305, "avg_output_tokens_per_task": 2441.375, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.44133320000000004, "accuracy_std_dev": 0.016000066667361108, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.101740272, "avg_duration_per_task_ms": 938564.588, "avg_output_tokens_per_task": 21680.24, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.44, "accuracy_std_dev": 0.02788886675718467, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03487082, "avg_duration_per_task_ms": 281485.155, "avg_output_tokens_per_task": 21754.5, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.43965518965517236, "accuracy_std_dev": 0.020525836824522862, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.045893882183908045, "avg_duration_per_task_ms": 128881.75862068965, "avg_output_tokens_per_task": 2035.2413793103449, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4387170412371134, "accuracy_std_dev": 0.022793011768351766, "run_count": 6, "total_questions": 291, "avg_cost_per_task_usd": 0.031083333333333338, "avg_duration_per_task_ms": 487106.5326460481, "avg_output_tokens_per_task": 4378.945017182131, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4383965367088608, "accuracy_std_dev": 0.03960836597472182, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.05240046329113924, "avg_duration_per_task_ms": 422067.2658227848, "avg_output_tokens_per_task": 20533.427848101266, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.43777799999999994, "accuracy_std_dev": 0.03861798340151904, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.19450383999999998, "avg_duration_per_task_ms": 50798.87333333334, "avg_output_tokens_per_task": 2047.5333333333333, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.011330936570096718, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02595394666666667, "avg_duration_per_task_ms": 66273.3, "avg_output_tokens_per_task": 2407.2933333333335, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21387673999999998, "avg_duration_per_task_ms": 47267.2, "avg_output_tokens_per_task": 2224.56, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.4373334, "accuracy_std_dev": 0.019136942750606727, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.077255652, "avg_duration_per_task_ms": 788193.684, "avg_output_tokens_per_task": 19596.964, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.4372223333333333, "accuracy_std_dev": 0.025270150254576823, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.027693089999999997, "avg_duration_per_task_ms": 78315.11, "avg_output_tokens_per_task": 2793.7166666666667, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.4370371111111111, "accuracy_std_dev": 0.04062616143828847, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.38184548444444444, "avg_duration_per_task_ms": 105579.94, "avg_output_tokens_per_task": 3104.1155555555556, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.4353740544217687, "accuracy_std_dev": 0.0333418611528944, "run_count": 6, "total_questions": 294, "avg_cost_per_task_usd": 0.03948419727891156, "avg_duration_per_task_ms": 53655.58163265306, "avg_output_tokens_per_task": 2073.0680272108843, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43416662500000003, "accuracy_std_dev": 0.01613729505630899, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2170375225, "avg_duration_per_task_ms": 46329.7025, "avg_output_tokens_per_task": 2265.4175, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.4333336666666667, "accuracy_std_dev": 0.0339934634239519, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03350491333333334, "avg_duration_per_task_ms": 439900.87333333335, "avg_output_tokens_per_task": 22781.906666666666, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.43333333333333335, "accuracy_std_dev": 0.014401440259764139, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03429177333333333, "avg_duration_per_task_ms": 515166.3933333333, "avg_output_tokens_per_task": 24569.606666666667, "last_run_timestamp": "2026-08-01T17:38:35.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.431111, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029769013333333334, "avg_duration_per_task_ms": 630152.1266666667, "avg_output_tokens_per_task": 18760.18666666667, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.24811575000000002, "avg_duration_per_task_ms": 46287.18, "avg_output_tokens_per_task": 2565.8, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.42888899999999996, "accuracy_std_dev": 0.022825013355235225, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07427926, "avg_duration_per_task_ms": 326411.7266666667, "avg_output_tokens_per_task": 3665.133333333333, "last_run_timestamp": "2026-10-01T19:04:28.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.4266666666666667, "accuracy_std_dev": 0.013052515015207695, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024687224444444444, "avg_duration_per_task_ms": 229766.70888888888, "avg_output_tokens_per_task": 3591.1422222222222, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.4256352959641255, "accuracy_std_dev": 0.029267857939983613, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.024396058295964126, "avg_duration_per_task_ms": 309280.5, "avg_output_tokens_per_task": 3539.0829596412555, "last_run_timestamp": "2026-10-03T16:12:07.465Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.4225002499999999, "accuracy_std_dev": 0.03843280391394179, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0546106075, "avg_duration_per_task_ms": 139402.745, "avg_output_tokens_per_task": 3558.66, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.41759616356877327, "accuracy_std_dev": 0.026805628250132423, "run_count": 11, "total_questions": 538, "avg_cost_per_task_usd": 0.0537584814126394, "avg_duration_per_task_ms": 212875.83271375464, "avg_output_tokens_per_task": 4238.7007434944235, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.4166667027027027, "accuracy_std_dev": 0.06529892179218752, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.23566864189189185, "avg_duration_per_task_ms": 251017.3918918919, "avg_output_tokens_per_task": 9861.692567567568, "last_run_timestamp": "2026-10-04T00:00:34.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.4162963333333333, "accuracy_std_dev": 0.12078393478715067, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02470130444444444, "avg_duration_per_task_ms": 179469.95555555556, "avg_output_tokens_per_task": 3348.8955555555553, "last_run_timestamp": "2026-10-03T16:09:19.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4136055102040816, "accuracy_std_dev": 0.02936042507338748, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.011379914285714287, "avg_duration_per_task_ms": 120011.6918367347, "avg_output_tokens_per_task": 3351.665306122449, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.4124148571428572, "accuracy_std_dev": 0.023430017383889733, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.03184153571428572, "avg_duration_per_task_ms": 707919.5331632653, "avg_output_tokens_per_task": 7547.298469387755, "last_run_timestamp": "2026-10-04T00:00:37.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.4092593333333333, "accuracy_std_dev": 0.022758969050073914, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06364212444444445, "avg_duration_per_task_ms": 99265.78666666667, "avg_output_tokens_per_task": 2928.4333333333334, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.4088887777777778, "accuracy_std_dev": 0.036514668117462, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05922039999999999, "avg_duration_per_task_ms": 309197.2711111111, "avg_output_tokens_per_task": 7366.042222222222, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40777783333333334, "accuracy_std_dev": 0.01781244190087991, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025491406666666664, "avg_duration_per_task_ms": 73232.91333333333, "avg_output_tokens_per_task": 2860.7966666666666, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.4053599698492462, "accuracy_std_dev": 0.008101744842933528, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.06672347236180905, "avg_duration_per_task_ms": 212067.50753768845, "avg_output_tokens_per_task": 8080.718592964824, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.4037037777777778, "accuracy_std_dev": 0.02436562151327963, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.026673933333333334, "avg_duration_per_task_ms": 283419.1955555556, "avg_output_tokens_per_task": 4179.831111111111, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.39999977777777784, "accuracy_std_dev": 0.02330672557094853, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07755904666666667, "avg_duration_per_task_ms": 122311.57111111112, "avg_output_tokens_per_task": 3122.713333333333, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.3994974723618091, "accuracy_std_dev": 0.15640288967474153, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.1472113216080402, "avg_duration_per_task_ms": 361190.4924623116, "avg_output_tokens_per_task": 52739.01005025126, "last_run_timestamp": "2026-10-03T16:22:23.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.3992593333333333, "accuracy_std_dev": 0.02801133907307301, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06248461111111111, "avg_duration_per_task_ms": 285179.9666666667, "avg_output_tokens_per_task": 10620.682222222222, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.39866660000000004, "accuracy_std_dev": 0.17082280826119212, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.016985011999999997, "avg_duration_per_task_ms": 130076.872, "avg_output_tokens_per_task": 3074.228, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.39851855555555554, "accuracy_std_dev": 0.04784532061900936, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.047124519999999996, "avg_duration_per_task_ms": 86880.75777777778, "avg_output_tokens_per_task": 3704.46, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.39703688888888894, "accuracy_std_dev": 0.020393928467095, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03641048222222223, "avg_duration_per_task_ms": 189988.64666666667, "avg_output_tokens_per_task": 3741.2244444444445, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.39666650000000003, "accuracy_std_dev": 0.02333349999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0319114, "avg_duration_per_task_ms": 79638.86, "avg_output_tokens_per_task": 5369.64, "last_run_timestamp": "2026-07-18T08:03:14.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.3948895644599303, "accuracy_std_dev": 0.014575970313613202, "run_count": 6, "total_questions": 287, "avg_cost_per_task_usd": 0.010113132404181184, "avg_duration_per_task_ms": 61320.738675958186, "avg_output_tokens_per_task": 1041.9094076655051, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.3944446666666666, "accuracy_std_dev": 0.015947545189011245, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07351260000000001, "avg_duration_per_task_ms": 297595.3333333333, "avg_output_tokens_per_task": 7950.953333333333, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.390508134529148, "accuracy_std_dev": 0.028796290448254745, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.060798977578475334, "avg_duration_per_task_ms": 369562.0852017937, "avg_output_tokens_per_task": 3902.7331838565024, "last_run_timestamp": "2026-10-03T16:12:24.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.39, "accuracy_std_dev": 0.014782396941407474, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028387520000000006, "avg_duration_per_task_ms": 78569.82333333333, "avg_output_tokens_per_task": 4258.516666666666, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.38814811111111114, "accuracy_std_dev": 0.0223944854097037, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02571107333333333, "avg_duration_per_task_ms": 215286.6288888889, "avg_output_tokens_per_task": 4475.344444444445, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.385833375, "accuracy_std_dev": 0.008457413743832989, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014628382499999999, "avg_duration_per_task_ms": 83726.5775, "avg_output_tokens_per_task": 599.935, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.38215508080808086, "accuracy_std_dev": 0.014660000000000006, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.05195866666666667, "avg_duration_per_task_ms": 134392.0202020202, "avg_output_tokens_per_task": 8684.282828282829, "last_run_timestamp": "2026-07-25T08:07:19.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.37978144444444445, "accuracy_std_dev": 0.018078801359923458, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.012327355191256831, "avg_duration_per_task_ms": 34864.229508196724, "avg_output_tokens_per_task": 609.8688524590164, "last_run_timestamp": "2026-10-03T16:12:33.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.3766665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07082534, "avg_duration_per_task_ms": 16764.74, "avg_output_tokens_per_task": 860.73, "last_run_timestamp": "2026-07-18T16:00:09.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.374074, "accuracy_std_dev": 0.03118154021211909, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05026599333333333, "avg_duration_per_task_ms": 71333.42666666667, "avg_output_tokens_per_task": 4374.031111111111, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.37166650000000007, "accuracy_std_dev": 0.01658317420610421, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.032970215000000004, "avg_duration_per_task_ms": 150852.9325, "avg_output_tokens_per_task": 4449.52, "last_run_timestamp": "2026-10-03T16:07:02.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.36476185714285714, "accuracy_std_dev": 0.029753169004943562, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07516260571428572, "avg_duration_per_task_ms": 18915.674285714285, "avg_output_tokens_per_task": 875.96, "last_run_timestamp": "2026-10-03T16:05:34.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3614163516483516, "accuracy_std_dev": 0.028501851452693475, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.070551663003663, "avg_duration_per_task_ms": 367421.45604395604, "avg_output_tokens_per_task": 7535.331501831502, "last_run_timestamp": "2026-10-03T08:32:03.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.3582909492385786, "accuracy_std_dev": 0.07485838274869422, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.0038857563451776644, "avg_duration_per_task_ms": 276001.0329949239, "avg_output_tokens_per_task": 4008.0659898477156, "last_run_timestamp": "2026-10-03T16:11:33.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.3564477189781022, "accuracy_std_dev": 0.023446226503206637, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.07946453284671531, "avg_duration_per_task_ms": 237669.46532846717, "avg_output_tokens_per_task": 7645.419708029197, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.3541665, "accuracy_std_dev": 0.015343212399950682, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.041681384999999994, "avg_duration_per_task_ms": 103610.79, "avg_output_tokens_per_task": 4507.515, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.35259277777777775, "accuracy_std_dev": 0.07615423091445964, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.005360093333333335, "avg_duration_per_task_ms": 130268.62, "avg_output_tokens_per_task": 3481.0555555555557, "last_run_timestamp": "2026-10-03T16:07:05.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.34353760714285714, "accuracy_std_dev": 0.24874446504633524, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.009734260204081633, "avg_duration_per_task_ms": 70782.52551020408, "avg_output_tokens_per_task": 2162.954081632653, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.34005986995515697, "accuracy_std_dev": 0.05113342105622878, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.02689973766816143, "avg_duration_per_task_ms": 99831.18609865471, "avg_output_tokens_per_task": 1162.4708520179372, "last_run_timestamp": "2026-10-03T16:14:36.877Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.3367119324324324, "accuracy_std_dev": 0.23078482961913535, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.014007192567567568, "avg_duration_per_task_ms": 1521898.9797297297, "avg_output_tokens_per_task": 94037.88851351352, "last_run_timestamp": "2026-10-03T16:12:24.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.3341665, "accuracy_std_dev": 0.0059509803184013355, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03100886, "avg_duration_per_task_ms": 84710.3, "avg_output_tokens_per_task": 3999.695, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.328889, "accuracy_std_dev": 0.022986915227291085, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08611619166666667, "avg_duration_per_task_ms": 152663.81666666668, "avg_output_tokens_per_task": 8236.506666666666, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.3206666, "accuracy_std_dev": 0.3296116389532384, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.019227021999999996, "avg_duration_per_task_ms": 58892.988, "avg_output_tokens_per_task": 10442.698, "last_run_timestamp": "2026-10-03T16:12:07.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.31282059999999995, "accuracy_std_dev": 0.030993226130882217, "run_count": 8, "total_questions": 390, "avg_cost_per_task_usd": 0.015985989743589744, "avg_duration_per_task_ms": 332807.73846153845, "avg_output_tokens_per_task": 4072.7410256410258, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.30370366666666665, "accuracy_std_dev": 0.035847625249101234, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16472546666666668, "avg_duration_per_task_ms": 120261.91777777778, "avg_output_tokens_per_task": 1852.1177777777777, "last_run_timestamp": "2026-10-03T16:16:35.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.2965195220883534, "accuracy_std_dev": 0.18696214545570447, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.14611226305220884, "avg_duration_per_task_ms": 854783.9919678714, "avg_output_tokens_per_task": 5611.467871485944, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.2844443333333333, "accuracy_std_dev": 0.031894788459698065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02081186, "avg_duration_per_task_ms": 62184.12666666666, "avg_output_tokens_per_task": 3804.7533333333336, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.28000025, "accuracy_std_dev": 0.027080189561513406, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02535222, "avg_duration_per_task_ms": 67904.845, "avg_output_tokens_per_task": 4023.05, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.266667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00574078, "avg_duration_per_task_ms": 359216.92, "avg_output_tokens_per_task": 2727.92, "last_run_timestamp": "2026-07-11T08:03:53.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2622223333333333, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08337879999999999, "avg_duration_per_task_ms": 576772.3666666667, "avg_output_tokens_per_task": 5220.566666666667, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24200919178082192, "accuracy_std_dev": 0.027381797668443095, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.07592458219178082, "avg_duration_per_task_ms": 576526.0342465753, "avg_output_tokens_per_task": 5262.068493150685, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.24, "accuracy_std_dev": 0.06798683968566596, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014107426666666664, "avg_duration_per_task_ms": 23182.855555555554, "avg_output_tokens_per_task": 1167.951111111111, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.23299353061224487, "accuracy_std_dev": 0.02687500000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.028852255102040817, "avg_duration_per_task_ms": 377353.28571428574, "avg_output_tokens_per_task": 5486.489795918367, "last_run_timestamp": "2026-07-18T08:04:09.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.2316665, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02960246, "avg_duration_per_task_ms": 418677.18, "avg_output_tokens_per_task": 5444.6, "last_run_timestamp": "2026-07-18T08:04:25.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.21466659999999996, "accuracy_std_dev": 0.04182530015959241, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.005141112, "avg_duration_per_task_ms": 40939.9, "avg_output_tokens_per_task": 1789.784, "last_run_timestamp": "2026-10-03T16:07:44.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.18766967073170732, "accuracy_std_dev": 0.018396157170452745, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.011055426829268291, "avg_duration_per_task_ms": 138140.34552845528, "avg_output_tokens_per_task": 5390.59756097561, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.17333325, "accuracy_std_dev": 0.11522926663368771, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0127158975, "avg_duration_per_task_ms": 62680.3375, "avg_output_tokens_per_task": 1280.3475, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.16858231034482757, "accuracy_std_dev": 0.10965908309383035, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.19030922126436778, "avg_duration_per_task_ms": 565575.5, "avg_output_tokens_per_task": 9876.801724137931, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.13333333333333333, "accuracy_std_dev": 0.021430473285072877, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007961373333333334, "avg_duration_per_task_ms": 80728.30333333333, "avg_output_tokens_per_task": 2469.17, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.11833325, "accuracy_std_dev": 0.018484159885899604, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007158584999999999, "avg_duration_per_task_ms": 55526.67, "avg_output_tokens_per_task": 2353.445, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.108889, "accuracy_std_dev": 0.008314764498569193, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006775626666666668, "avg_duration_per_task_ms": 60620.74, "avg_output_tokens_per_task": 2327.0066666666667, "last_run_timestamp": "2026-10-01T19:04:36.194Z" } ] }