{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-10-10", "retrieved_at": "2026-10-10T01:45:23.153Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2157, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9483723333333332, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20306945454545455, "avg_duration_per_task_ms": 134835.03535353535, "avg_output_tokens_per_task": 16798.409090909092, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021931015151515153, "avg_duration_per_task_ms": 68600.9191919192, "avg_output_tokens_per_task": 1835.4242424242425, "last_run_timestamp": "2026-10-03T01:13:05.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.947009, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.07090916923076923, "avg_duration_per_task_ms": 142045.61538461538, "avg_output_tokens_per_task": 18762.907692307694, "last_run_timestamp": "2026-10-01T08:30:23.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11987038888888889, "avg_duration_per_task_ms": 35572.169191919194, "avg_output_tokens_per_task": 2020.9419191919192, "last_run_timestamp": "2026-10-01T08:24:02.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.015926459595959598, "avg_duration_per_task_ms": 24665.858585858587, "avg_output_tokens_per_task": 2219.030303030303, "last_run_timestamp": "2026-09-28T18:11:34.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.9393935, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.29793838636363634, "avg_duration_per_task_ms": 106861.7196969697, "avg_output_tokens_per_task": 11692.656565656565, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.937943329787234, "accuracy_std_dev": 0.011200313606720516, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.023761370567375886, "avg_duration_per_task_ms": 52418.78723404255, "avg_output_tokens_per_task": 7636.52304964539, "last_run_timestamp": "2026-10-01T16:16:37.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9371493333333333, "accuracy_std_dev": 0.004199386807089248, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.30925494444444446, "avg_duration_per_task_ms": 152328.37037037036, "avg_output_tokens_per_task": 10179.092592592593, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.9337823333333333, "accuracy_std_dev": 0.008398773614178557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3842955404040404, "avg_duration_per_task_ms": 82918.1936026936, "avg_output_tokens_per_task": 11996.59090909091, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9332206666666667, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.24476150168350166, "avg_duration_per_task_ms": 101605.01851851853, "avg_output_tokens_per_task": 11508.424242424242, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.932951424427481, "accuracy_std_dev": 0.006875320605511528, "run_count": 7, "total_questions": 1310, "avg_cost_per_task_usd": 0.13016139770992366, "avg_duration_per_task_ms": 164770.279389313, "avg_output_tokens_per_task": 8479.913740458016, "last_run_timestamp": "2026-10-09T00:00:06.212Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9326596250000001, "accuracy_std_dev": 0.00862529761714779, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.32610833964646463, "avg_duration_per_task_ms": 161836.85416666666, "avg_output_tokens_per_task": 10112.390782828283, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.9326595000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02474653787878788, "avg_duration_per_task_ms": 20002.444444444445, "avg_output_tokens_per_task": 2234.5, "last_run_timestamp": "2026-10-03T09:05:12.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9325395714285714, "accuracy_std_dev": 0.004401689845956867, "run_count": 4, "total_questions": 756, "avg_cost_per_task_usd": 0.06429072222222224, "avg_duration_per_task_ms": 63071.10317460317, "avg_output_tokens_per_task": 11529.871693121693, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.931257, "accuracy_std_dev": 0.00924235327175929, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.026367194444444447, "avg_duration_per_task_ms": 19064.280303030304, "avg_output_tokens_per_task": 2249.5185185185187, "last_run_timestamp": "2026-10-03T09:39:59.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.930415, "accuracy_std_dev": 0.004827415699246697, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06216489141414142, "avg_duration_per_task_ms": 64761.851851851854, "avg_output_tokens_per_task": 11730.429292929293, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9278394565075213, "accuracy_std_dev": 0.008041238261262993, "run_count": 25, "total_questions": 4587, "avg_cost_per_task_usd": 0.1608165642031829, "avg_duration_per_task_ms": 149318.05646391978, "avg_output_tokens_per_task": 8864.42119032047, "last_run_timestamp": "2026-10-03T10:48:13.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9277417729032257, "accuracy_std_dev": 0.015966818205265555, "run_count": 5, "total_questions": 775, "avg_cost_per_task_usd": 0.09847772, "avg_duration_per_task_ms": 244962.84516129032, "avg_output_tokens_per_task": 7582.7083870967745, "last_run_timestamp": "2026-10-03T10:05:38.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.004418715424192887, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05958464141414141, "avg_duration_per_task_ms": 58983.43771043771, "avg_output_tokens_per_task": 11740.584175084176, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.926400137295082, "accuracy_std_dev": 0.007132075389394045, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09721093032786886, "avg_duration_per_task_ms": 197311.04713114753, "avg_output_tokens_per_task": 8911.119877049181, "last_run_timestamp": "2026-10-03T10:09:17.601Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.925926, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025347974747474744, "avg_duration_per_task_ms": 21521.87373737374, "avg_output_tokens_per_task": 2298.931818181818, "last_run_timestamp": "2026-10-03T09:35:22.953Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.925926, "accuracy_std_dev": 0.005498287975967322, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1441699393939394, "avg_duration_per_task_ms": 61658.986531986535, "avg_output_tokens_per_task": 15894.8367003367, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.925926, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.14454752525252526, "avg_duration_per_task_ms": 66158.0808080808, "avg_output_tokens_per_task": 9383.212121212122, "last_run_timestamp": "2026-10-03T19:50:41.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9259256666666668, "accuracy_std_dev": 0.0036366213562713575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1413088872053872, "avg_duration_per_task_ms": 82468.60606060606, "avg_output_tokens_per_task": 15584.752525252525, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9259255000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025620611111111115, "avg_duration_per_task_ms": 19140.497474747473, "avg_output_tokens_per_task": 2323.35101010101, "last_run_timestamp": "2026-10-03T09:34:06.834Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9255440309278352, "accuracy_std_dev": 0.0047577793839843644, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.3407643092783505, "avg_duration_per_task_ms": 132309.46391752578, "avg_output_tokens_per_task": 10197.671821305841, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9248735, "accuracy_std_dev": 0.00862277742377712, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10364464204545455, "avg_duration_per_task_ms": 55123.33648989899, "avg_output_tokens_per_task": 3517.8642676767677, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9242421666666667, "accuracy_std_dev": 0.0048599613653014136, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.14878865656565657, "avg_duration_per_task_ms": 85040.50420875421, "avg_output_tokens_per_task": 15641.649831649831, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto", "model_slug": "unbiased/pareto", "model_name": "Pareto", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03377419696969697, "avg_duration_per_task_ms": 114763.54545454546, "avg_output_tokens_per_task": 4358.363636363636, "last_run_timestamp": "2026-09-29T03:11:33.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.008625346413333228, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05533751388888889, "avg_duration_per_task_ms": 61705.84722222222, "avg_output_tokens_per_task": 3564.286616161616, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9227054347826086, "accuracy_std_dev": 0.0037342422507786015, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.09927257048748353, "avg_duration_per_task_ms": 52555.73781291173, "avg_output_tokens_per_task": 3495.6416337285905, "last_run_timestamp": "2026-10-01T19:12:57.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9219427970033833, "accuracy_std_dev": 0.006990580536078893, "run_count": 11, "total_questions": 2069, "avg_cost_per_task_usd": 0.15209735766070562, "avg_duration_per_task_ms": 132397.76655389077, "avg_output_tokens_per_task": 9993.401159980667, "last_run_timestamp": "2026-10-03T10:21:23.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9212965, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13693505808080808, "avg_duration_per_task_ms": 189958.2702020202, "avg_output_tokens_per_task": 8925.60101010101, "last_run_timestamp": "2026-10-03T09:18:30.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06043574579124578, "avg_duration_per_task_ms": 62971.38047138047, "avg_output_tokens_per_task": 3586.8703703703704, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.920875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025261131313131315, "avg_duration_per_task_ms": 21541.484848484848, "avg_output_tokens_per_task": 2294.9949494949497, "last_run_timestamp": "2026-10-03T09:35:43.283Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9205012222222222, "accuracy_std_dev": 0.00850227984821038, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1576766610549944, "avg_duration_per_task_ms": 236410.12177328844, "avg_output_tokens_per_task": 12595.254208754208, "last_run_timestamp": "2026-09-26T08:55:21.775Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.920314, "accuracy_std_dev": 0.007316769300722847, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1692038207070707, "avg_duration_per_task_ms": 320867.6077441077, "avg_output_tokens_per_task": 9628.20622895623, "last_run_timestamp": "2026-10-03T09:31:32.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026391550505050505, "avg_duration_per_task_ms": 26392.646464646463, "avg_output_tokens_per_task": 2240.712121212121, "last_run_timestamp": "2026-10-01T08:37:17.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9185433141891891, "accuracy_std_dev": 0.011935651348422994, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.13230360641891895, "avg_duration_per_task_ms": 318469.4380630631, "avg_output_tokens_per_task": 10386.162725225226, "last_run_timestamp": "2026-10-03T09:44:31.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9181816, "accuracy_std_dev": 0.007721961838807562, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0816559797979798, "avg_duration_per_task_ms": 327979.47777777776, "avg_output_tokens_per_task": 24710.48888888889, "last_run_timestamp": "2026-10-03T11:44:39.356Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9173829326647566, "accuracy_std_dev": 0.013022336741090185, "run_count": 11, "total_questions": 2094, "avg_cost_per_task_usd": 0.12426453438395414, "avg_duration_per_task_ms": 122014.65329512894, "avg_output_tokens_per_task": 8324.876313276027, "last_run_timestamp": "2026-10-03T09:17:37.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9171245769230769, "accuracy_std_dev": 0.013071755977424514, "run_count": 6, "total_questions": 728, "avg_cost_per_task_usd": 0.1094587857142857, "avg_duration_per_task_ms": 198227.6401098901, "avg_output_tokens_per_task": 7265.833791208791, "last_run_timestamp": "2026-10-09T16:12:14.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9161053333333333, "accuracy_std_dev": 0.004900048423797018, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10286850841750843, "avg_duration_per_task_ms": 630668.6658249159, "avg_output_tokens_per_task": 39966.40488215488, "last_run_timestamp": "2026-10-03T09:15:20.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9158830997830802, "accuracy_std_dev": 0.0087081660666812, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.2615176702819957, "avg_duration_per_task_ms": 379032.86912509036, "avg_output_tokens_per_task": 37727.92263195951, "last_run_timestamp": "2026-10-03T09:17:56.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9153725336438923, "accuracy_std_dev": 0.007489152329202574, "run_count": 10, "total_questions": 1932, "avg_cost_per_task_usd": 0.14073902795031057, "avg_duration_per_task_ms": 293484.7536231884, "avg_output_tokens_per_task": 9227.590062111802, "last_run_timestamp": "2026-10-03T10:17:13.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9148461928609483, "accuracy_std_dev": 0.040300990523310966, "run_count": 10, "total_questions": 1877, "avg_cost_per_task_usd": 0.11435862759722962, "avg_duration_per_task_ms": 215504.55833777305, "avg_output_tokens_per_task": 9109.937133724028, "last_run_timestamp": "2026-10-03T10:08:37.048Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9145619999999999, "accuracy_std_dev": 0.003005841230005351, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025954439393939394, "avg_duration_per_task_ms": 19982.071969696968, "avg_output_tokens_per_task": 2234.438131313131, "last_run_timestamp": "2026-10-03T10:12:04.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9141412727272726, "accuracy_std_dev": 0.01313873065053996, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0326499334251607, "avg_duration_per_task_ms": 226257.22359963268, "avg_output_tokens_per_task": 27026.729568411385, "last_run_timestamp": "2026-10-03T16:51:25.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9132995, "accuracy_std_dev": 0.010546877796295947, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.15848487878787879, "avg_duration_per_task_ms": 520006.99494949495, "avg_output_tokens_per_task": 39704.125, "last_run_timestamp": "2026-10-03T09:15:53.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9117844, "accuracy_std_dev": 0.006079420781620585, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03790721616161616, "avg_duration_per_task_ms": 129456.97070707071, "avg_output_tokens_per_task": 28484.87575757576, "last_run_timestamp": "2026-10-03T09:28:48.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9115787584210527, "accuracy_std_dev": 0.03150249392869556, "run_count": 20, "total_questions": 3800, "avg_cost_per_task_usd": 0.14629828526315786, "avg_duration_per_task_ms": 443992.2734210526, "avg_output_tokens_per_task": 8936.269473684211, "last_run_timestamp": "2026-10-03T10:26:01.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9098249433962265, "accuracy_std_dev": 0.01556456113130081, "run_count": 10, "total_questions": 1961, "avg_cost_per_task_usd": 0.18924138296787354, "avg_duration_per_task_ms": 516166.16930137685, "avg_output_tokens_per_task": 12560.821009688934, "last_run_timestamp": "2026-10-03T08:12:32.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9095505370786516, "accuracy_std_dev": 0.004402180380080482, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.02965778258426966, "avg_duration_per_task_ms": 474993.9230337079, "avg_output_tokens_per_task": 30670.006179775282, "last_run_timestamp": "2026-10-03T17:37:28.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9094274, "accuracy_std_dev": 0.002474202384608023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07117444141414142, "avg_duration_per_task_ms": 135218.17474747475, "avg_output_tokens_per_task": 27695.128282828282, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0054125, "avg_duration_per_task_ms": 235839.91414141413, "avg_output_tokens_per_task": 10749.717171717171, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9075092967032967, "accuracy_std_dev": 0.0008110000000000062, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.15048144505494504, "avg_duration_per_task_ms": 345399.21428571426, "avg_output_tokens_per_task": 38131.52197802198, "last_run_timestamp": "2026-10-03T10:04:03.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.907407, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11648837373737374, "avg_duration_per_task_ms": 581221.1060606061, "avg_output_tokens_per_task": 46418.58585858586, "last_run_timestamp": "2026-10-03T09:31:25.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.9072202222222222, "accuracy_std_dev": 0.017575364676335258, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13183432884399549, "avg_duration_per_task_ms": 553846.1520763187, "avg_output_tokens_per_task": 39640.39955106622, "last_run_timestamp": "2026-10-09T21:00:06.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9071381724573594, "accuracy_std_dev": 0.03371213456235009, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.015826173089071385, "avg_duration_per_task_ms": 64047.12571067593, "avg_output_tokens_per_task": 3604.423246999368, "last_run_timestamp": "2026-09-30T03:11:21.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9071160662921349, "accuracy_std_dev": 0.01732392826250572, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.03509260337078652, "avg_duration_per_task_ms": 322629.44325842697, "avg_output_tokens_per_task": 29224.32752808989, "last_run_timestamp": "2026-10-03T19:03:35.841Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9063971, "accuracy_std_dev": 0.022689051418911293, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.03762983484848485, "avg_duration_per_task_ms": 268783.10353535356, "avg_output_tokens_per_task": 31583.5601010101, "last_run_timestamp": "2026-10-03T16:39:40.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9060606, "accuracy_std_dev": 0.006060466668500054, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.023695626262626262, "avg_duration_per_task_ms": 214413.58686868686, "avg_output_tokens_per_task": 29199.484848484848, "last_run_timestamp": "2026-10-03T09:22:46.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9059472448132779, "accuracy_std_dev": 0.025925182844816933, "run_count": 11, "total_questions": 2169, "avg_cost_per_task_usd": 0.03841674366067312, "avg_duration_per_task_ms": 255822.2120792992, "avg_output_tokens_per_task": 30855.921161825725, "last_run_timestamp": "2026-10-03T16:57:18.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.90572375, "accuracy_std_dev": 0.009744176680843784, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03296931691919192, "avg_duration_per_task_ms": 228601.25883838383, "avg_output_tokens_per_task": 27296.915404040403, "last_run_timestamp": "2026-10-03T10:17:41.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.905492966223132, "accuracy_std_dev": 0.012089569061385095, "run_count": 10, "total_questions": 1954, "avg_cost_per_task_usd": 0.02905465660184237, "avg_duration_per_task_ms": 262994.571136131, "avg_output_tokens_per_task": 24078.712896622314, "last_run_timestamp": "2026-10-03T16:51:44.428Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9053872, "accuracy_std_dev": 0.006933048201188252, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.035172772727272726, "avg_duration_per_task_ms": 190581.2101010101, "avg_output_tokens_per_task": 29132.872727272726, "last_run_timestamp": "2026-10-03T09:10:43.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.9053669934810953, "accuracy_std_dev": 0.014124276768541987, "run_count": 8, "total_questions": 1534, "avg_cost_per_task_usd": 0.012634106910039114, "avg_duration_per_task_ms": 286058.7731421121, "avg_output_tokens_per_task": 17810.844198174706, "last_run_timestamp": "2026-10-03T10:17:33.683Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9051339848275863, "accuracy_std_dev": 0.019662141676895572, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.033351784367816086, "avg_duration_per_task_ms": 333114.86988505744, "avg_output_tokens_per_task": 29114.81333333333, "last_run_timestamp": "2026-10-03T18:02:12.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.903872, "accuracy_std_dev": 0.028683231540396564, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04186602929292929, "avg_duration_per_task_ms": 291657.99141414143, "avg_output_tokens_per_task": 32019.437373737375, "last_run_timestamp": "2026-10-03T16:15:35.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.903424887142055, "accuracy_std_dev": 0.056166981894488544, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.14229890510948906, "avg_duration_per_task_ms": 308136.8270634475, "avg_output_tokens_per_task": 10189.49241998877, "last_run_timestamp": "2026-10-03T10:42:35.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9031985, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.033975815656565655, "avg_duration_per_task_ms": 141490.74494949495, "avg_output_tokens_per_task": 28136.507575757576, "last_run_timestamp": "2026-10-03T09:38:38.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902918, "accuracy_std_dev": 0.0007933738084913016, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14672305387205387, "avg_duration_per_task_ms": 133620.78956228957, "avg_output_tokens_per_task": 9652.131313131313, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.9028655747851002, "accuracy_std_dev": 0.042913716403447925, "run_count": 9, "total_questions": 1745, "avg_cost_per_task_usd": 0.010968459025787967, "avg_duration_per_task_ms": 489384.2842406877, "avg_output_tokens_per_task": 20816.36446991404, "last_run_timestamp": "2026-10-03T09:16:27.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9027775, "accuracy_std_dev": 0.015400856096334393, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.026259813131313134, "avg_duration_per_task_ms": 203049.0542929293, "avg_output_tokens_per_task": 26132.907828282827, "last_run_timestamp": "2026-10-09T21:02:25.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9024267704752275, "accuracy_std_dev": 0.008260579897319528, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.03290473609706774, "avg_duration_per_task_ms": 147318.88675429727, "avg_output_tokens_per_task": 29037.182002022244, "last_run_timestamp": "2026-10-03T09:38:12.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.9023569999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05417848484848485, "avg_duration_per_task_ms": 36580.51515151515, "avg_output_tokens_per_task": 2499.1969696969695, "last_run_timestamp": "2026-10-03T09:27:36.763Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9023565, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.032186345959595956, "avg_duration_per_task_ms": 777530.196969697, "avg_output_tokens_per_task": 36633.931818181816, "last_run_timestamp": "2026-10-03T10:59:16.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9021696666666666, "accuracy_std_dev": 0.023279077501768247, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15281446520763184, "avg_duration_per_task_ms": 592828.4741863075, "avg_output_tokens_per_task": 38634.81593714927, "last_run_timestamp": "2026-10-03T10:13:07.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9017359124999998, "accuracy_std_dev": 0.023311648922467388, "run_count": 8, "total_questions": 1440, "avg_cost_per_task_usd": 0.15431451527777776, "avg_duration_per_task_ms": 567983.5055555556, "avg_output_tokens_per_task": 37740.19097222222, "last_run_timestamp": "2026-10-03T09:09:53.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9016834000000001, "accuracy_std_dev": 0.04222344006165297, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1424910212121212, "avg_duration_per_task_ms": 211878.58636363636, "avg_output_tokens_per_task": 9531.086363636363, "last_run_timestamp": "2026-10-03T10:10:50.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.9016834, "accuracy_std_dev": 0.01216780761846602, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.028648670707070705, "avg_duration_per_task_ms": 147445.17373737373, "avg_output_tokens_per_task": 28162.69292929293, "last_run_timestamp": "2026-10-03T09:28:15.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9014598562605278, "accuracy_std_dev": 0.03309502797836693, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.10678652554744525, "avg_duration_per_task_ms": 569188.9988770353, "avg_output_tokens_per_task": 39083.0421111735, "last_run_timestamp": "2026-10-03T10:32:36.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.9007667777777778, "accuracy_std_dev": 0.005624838482180366, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.20504940852974188, "avg_duration_per_task_ms": 98780.56790123456, "avg_output_tokens_per_task": 7889.076038159372, "last_run_timestamp": "2026-10-03T09:47:38.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.900635386539607, "accuracy_std_dev": 0.009720906385723498, "run_count": 10, "total_questions": 1679, "avg_cost_per_task_usd": 0.022405076235854672, "avg_duration_per_task_ms": 187927.48243001787, "avg_output_tokens_per_task": 18532.87313877308, "last_run_timestamp": "2026-10-03T16:51:49.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9004328214285713, "accuracy_std_dev": 0.005913167322978487, "run_count": 28, "total_questions": 5544, "avg_cost_per_task_usd": 0.21622311616161613, "avg_duration_per_task_ms": 94665.56475468975, "avg_output_tokens_per_task": 7881.3840187590185, "last_run_timestamp": "2026-10-03T10:10:31.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9001119999999999, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.053618132996633, "avg_duration_per_task_ms": 27125.141414141413, "avg_output_tokens_per_task": 2464.4410774410776, "last_run_timestamp": "2026-10-03T09:16:11.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8999998999999999, "accuracy_std_dev": 0.005644066804175849, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.012073083333333333, "avg_duration_per_task_ms": 179513.8106060606, "avg_output_tokens_per_task": 26194.345454545455, "last_run_timestamp": "2026-10-03T09:29:36.040Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8999248888888889, "accuracy_std_dev": 0.006354339085213833, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15479229124579125, "avg_duration_per_task_ms": 130090.1857463524, "avg_output_tokens_per_task": 9547.675645342311, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8998314999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0343437398989899, "avg_duration_per_task_ms": 823628.1161616162, "avg_output_tokens_per_task": 39114.656565656565, "last_run_timestamp": "2026-10-03T11:02:12.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.8995779088607595, "accuracy_std_dev": 0.007113645637083682, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.016700413502109705, "avg_duration_per_task_ms": 439475.2455696203, "avg_output_tokens_per_task": 29074.91983122363, "last_run_timestamp": "2026-10-08T23:08:23.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14109027272727273, "avg_duration_per_task_ms": 131303, "avg_output_tokens_per_task": 9278.422558922559, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0412280202020202, "avg_duration_per_task_ms": 141835.40572390571, "avg_output_tokens_per_task": 27211.210437710437, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8994104999999998, "accuracy_std_dev": 0.004817079950550935, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.22572103156565654, "avg_duration_per_task_ms": 108993.70707070707, "avg_output_tokens_per_task": 8821.869949494949, "last_run_timestamp": "2026-10-03T09:42:54.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8992219655172418, "accuracy_std_dev": 0.0066569301119999745, "run_count": 29, "total_questions": 5742, "avg_cost_per_task_usd": 0.22289186381051904, "avg_duration_per_task_ms": 94799.24817136886, "avg_output_tokens_per_task": 8115.898293277603, "last_run_timestamp": "2026-10-03T10:16:05.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8991767777777778, "accuracy_std_dev": 0.005290828285875197, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.058499294612794614, "avg_duration_per_task_ms": 32080.961840628508, "avg_output_tokens_per_task": 2531.0488215488217, "last_run_timestamp": "2026-10-03T10:16:57.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.005188652155184398, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03249949747474747, "avg_duration_per_task_ms": 137621.66666666666, "avg_output_tokens_per_task": 26156.715909090908, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna", "model_slug": "openai/gpt-6-luna", "model_name": "GPT-6-Luna", "provider_name": "auto-routing", "accuracy": 0.898477, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.003442197969543147, "avg_duration_per_task_ms": 73312.30456852792, "avg_output_tokens_per_task": 6831.9390862944165, "last_run_timestamp": "2026-10-08T13:29:57.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8980803802559414, "accuracy_std_dev": 0.010966029181066412, "run_count": 6, "total_questions": 1094, "avg_cost_per_task_usd": 0.006137671846435101, "avg_duration_per_task_ms": 419028.80987202923, "avg_output_tokens_per_task": 12654.702925045704, "last_run_timestamp": "2026-10-03T10:14:40.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8980545555555557, "accuracy_std_dev": 0.0033774129188909115, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.057495572951739626, "avg_duration_per_task_ms": 25176.90740740741, "avg_output_tokens_per_task": 2488.7480359147025, "last_run_timestamp": "2026-10-03T09:48:22.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.8978673333333335, "accuracy_std_dev": 0.005382399950042904, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.057627444444444446, "avg_duration_per_task_ms": 26162.272727272728, "avg_output_tokens_per_task": 2526.015993265993, "last_run_timestamp": "2026-10-03T10:06:47.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8977272500000001, "accuracy_std_dev": 0.008615029218609743, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19720289898989898, "avg_duration_per_task_ms": 95317.51073232324, "avg_output_tokens_per_task": 7684.227904040404, "last_run_timestamp": "2026-10-03T09:39:18.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.8975330208728654, "accuracy_std_dev": 0.015763870376592163, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.11425380961416824, "avg_duration_per_task_ms": 628521.0765338393, "avg_output_tokens_per_task": 36072.062618595824, "last_run_timestamp": "2026-10-03T10:20:39.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8974932222222222, "accuracy_std_dev": 0.012382476271622079, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.20160183445566776, "avg_duration_per_task_ms": 97321.57126823794, "avg_output_tokens_per_task": 7848.616722783389, "last_run_timestamp": "2026-10-03T09:45:23.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8968568964143425, "accuracy_std_dev": 0.005278733293840445, "run_count": 4, "total_questions": 753, "avg_cost_per_task_usd": 0.15754132403718457, "avg_duration_per_task_ms": 84099.57768924303, "avg_output_tokens_per_task": 9420.674634794157, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.8967969786476867, "accuracy_std_dev": 0.018458871221177085, "run_count": 10, "total_questions": 1967, "avg_cost_per_task_usd": 0.11560783629893238, "avg_duration_per_task_ms": 391555.7447890188, "avg_output_tokens_per_task": 7906.456532791052, "last_run_timestamp": "2026-10-08T22:32:00.376Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8967218206896553, "accuracy_std_dev": 0.007385261912078655, "run_count": 20, "total_questions": 3915, "avg_cost_per_task_usd": 0.12559540842911876, "avg_duration_per_task_ms": 57752.34942528736, "avg_output_tokens_per_task": 4741.332567049809, "last_run_timestamp": "2026-10-03T10:05:55.795Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8966562765957448, "accuracy_std_dev": 0.007814351796534355, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.018003054711246202, "avg_duration_per_task_ms": 581928.556231003, "avg_output_tokens_per_task": 17806.63627152989, "last_run_timestamp": "2026-10-05T20:49:39.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.8962022556962025, "accuracy_std_dev": 0.0022689999999999655, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.020123096202531644, "avg_duration_per_task_ms": 653583.2962025317, "avg_output_tokens_per_task": 42659.516455696205, "last_run_timestamp": "2026-10-10T00:00:04.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8960818181818182, "accuracy_std_dev": 0.021738909414052866, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.08613982369146005, "avg_duration_per_task_ms": 211123.70798898072, "avg_output_tokens_per_task": 28935.424242424244, "last_run_timestamp": "2026-10-05T18:10:18.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8956340674536255, "accuracy_std_dev": 0.02132427030288766, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.13012930916245083, "avg_duration_per_task_ms": 507416.96458684653, "avg_output_tokens_per_task": 37964.30185497471, "last_run_timestamp": "2026-10-03T10:12:16.841Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8956230000000002, "accuracy_std_dev": 0.007273551310513056, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00771163468013468, "avg_duration_per_task_ms": 254636.3367003367, "avg_output_tokens_per_task": 27268.55387205387, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8956227499999999, "accuracy_std_dev": 0.005791223203046117, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.20478679124579122, "avg_duration_per_task_ms": 99940.12121212122, "avg_output_tokens_per_task": 7979.755892255892, "last_run_timestamp": "2026-10-03T09:40:16.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8952778724516466, "accuracy_std_dev": 0.027063673618015708, "run_count": 11, "total_questions": 1913, "avg_cost_per_task_usd": 0.2081439644537376, "avg_duration_per_task_ms": 408679.8609513853, "avg_output_tokens_per_task": 45592.24150548876, "last_run_timestamp": "2026-10-08T06:00:12.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8951176, "accuracy_std_dev": 0.00756834082477789, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.12048651767676768, "avg_duration_per_task_ms": 57579.538383838386, "avg_output_tokens_per_task": 4603.499494949495, "last_run_timestamp": "2026-10-03T09:25:15.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8950130708661417, "accuracy_std_dev": 0.00891334037763057, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.048151682414698166, "avg_duration_per_task_ms": 43476.24934383202, "avg_output_tokens_per_task": 3503.036745406824, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8949103109177213, "accuracy_std_dev": 0.025790262867514752, "run_count": 7, "total_questions": 1264, "avg_cost_per_task_usd": 0.02656089082278481, "avg_duration_per_task_ms": 720996.8117088608, "avg_output_tokens_per_task": 24102.247626582277, "last_run_timestamp": "2026-09-12T16:17:57.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8948596495327101, "accuracy_std_dev": 0.0171199232602798, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.022189622274143304, "avg_duration_per_task_ms": 231984.90109034267, "avg_output_tokens_per_task": 26737.70015576324, "last_run_timestamp": "2026-10-03T09:42:10.726Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8945527678958785, "accuracy_std_dev": 0.010412875331697246, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.012522617136659437, "avg_duration_per_task_ms": 291164.3582791034, "avg_output_tokens_per_task": 27224.147143890095, "last_run_timestamp": "2026-10-03T09:48:09.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8942841547799696, "accuracy_std_dev": 0.008875212006481873, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.22388517248356093, "avg_duration_per_task_ms": 107507.08952959029, "avg_output_tokens_per_task": 8731.16894284269, "last_run_timestamp": "2026-10-03T09:18:11.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8941264444444444, "accuracy_std_dev": 0.009636502052221474, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05178651795735129, "avg_duration_per_task_ms": 45445.343434343435, "avg_output_tokens_per_task": 3671.5897867564536, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8938831666666668, "accuracy_std_dev": 0.007122347684499023, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.20402719309764314, "avg_duration_per_task_ms": 70642.39461279461, "avg_output_tokens_per_task": 7430.633333333333, "last_run_timestamp": "2026-10-03T10:09:58.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.89267675, "accuracy_std_dev": 0.006979218773437292, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037834488636363635, "avg_duration_per_task_ms": 50710.07575757576, "avg_output_tokens_per_task": 3480.1502525252527, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.89267063172226, "accuracy_std_dev": 0.013806389148989432, "run_count": 15, "total_questions": 2938, "avg_cost_per_task_usd": 0.00379094996596324, "avg_duration_per_task_ms": 310296.7300884956, "avg_output_tokens_per_task": 20769.534717494895, "last_run_timestamp": "2026-10-08T23:06:17.060Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.892548239520958, "accuracy_std_dev": 0.006915975724525221, "run_count": 3, "total_questions": 501, "avg_cost_per_task_usd": 0.013722602794411178, "avg_duration_per_task_ms": 355406.4910179641, "avg_output_tokens_per_task": 23503.43512974052, "last_run_timestamp": "2026-10-03T10:04:24.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.892405, "accuracy_std_dev": null, "run_count": 1, "total_questions": 158, "avg_cost_per_task_usd": 0.12755274683544304, "avg_duration_per_task_ms": 1596411.9683544303, "avg_output_tokens_per_task": 50780.917721518985, "last_run_timestamp": "2026-10-08T23:19:46.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8922556666666667, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05459258585858586, "avg_duration_per_task_ms": 34041.639730639734, "avg_output_tokens_per_task": 2518.429292929293, "last_run_timestamp": "2026-10-03T09:50:45.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.8909751541501977, "accuracy_std_dev": 0.003207768937369934, "run_count": 3, "total_questions": 506, "avg_cost_per_task_usd": 0.0325840256916996, "avg_duration_per_task_ms": 936484.3814229249, "avg_output_tokens_per_task": 37243.156126482216, "last_run_timestamp": "2026-10-08T22:57:13.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8905723333333333, "accuracy_std_dev": 0.021339306148253502, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13216584343434343, "avg_duration_per_task_ms": 251067.54713804714, "avg_output_tokens_per_task": 43934.003367003366, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.890572, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12533664393939395, "avg_duration_per_task_ms": 808381.0631313132, "avg_output_tokens_per_task": 49868.60353535353, "last_run_timestamp": "2026-10-03T09:33:15.909Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8904339810366624, "accuracy_std_dev": 0.010766495576556007, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.020936142857142858, "avg_duration_per_task_ms": 384871.00505689, "avg_output_tokens_per_task": 23969.699115044248, "last_run_timestamp": "2026-10-09T02:24:39.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.005722799567422281, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.19800758417508418, "avg_duration_per_task_ms": 96141.14141414141, "avg_output_tokens_per_task": 7695.558922558923, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8900110000000001, "accuracy_std_dev": 0.0055552329084086844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2164912121212121, "avg_duration_per_task_ms": 104660.01851851853, "avg_output_tokens_per_task": 8434.90404040404, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8898507142857144, "accuracy_std_dev": 0.013463734020527524, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.013538019841269842, "avg_duration_per_task_ms": 210147.86976911977, "avg_output_tokens_per_task": 24623.09126984127, "last_run_timestamp": "2026-10-03T09:29:34.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8898480650076179, "accuracy_std_dev": 0.00936748461292215, "run_count": 30, "total_questions": 5907, "avg_cost_per_task_usd": 0.13325874741831725, "avg_duration_per_task_ms": 59111.40138818351, "avg_output_tokens_per_task": 4767.174030810902, "last_run_timestamp": "2026-10-03T09:22:12.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8895201249999999, "accuracy_std_dev": 0.018150193638343783, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1383345025252525, "avg_duration_per_task_ms": 580496.523989899, "avg_output_tokens_per_task": 34909.79734848485, "last_run_timestamp": "2026-10-03T10:33:13.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.88945, "accuracy_std_dev": 0.00975207530050226, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.024394900673400673, "avg_duration_per_task_ms": 319196.3552188552, "avg_output_tokens_per_task": 27886, "last_run_timestamp": "2026-10-03T10:36:49.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8894080654205608, "accuracy_std_dev": 0.00941594536414648, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.21110783605919006, "avg_duration_per_task_ms": 101949.81853582554, "avg_output_tokens_per_task": 8216.18808411215, "last_run_timestamp": "2026-10-03T09:19:25.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8892253999999999, "accuracy_std_dev": 0.008511315206241623, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.24073908914141412, "avg_duration_per_task_ms": 84900.21515151515, "avg_output_tokens_per_task": 8956.617424242424, "last_run_timestamp": "2026-10-03T09:30:01.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8888886666666669, "accuracy_std_dev": 0.007653072992087717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008966420875420876, "avg_duration_per_task_ms": 148767.4797979798, "avg_output_tokens_per_task": 31547.74074074074, "last_run_timestamp": "2026-10-03T10:11:42.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8883276666666667, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008124902356902356, "avg_duration_per_task_ms": 491163.50673400675, "avg_output_tokens_per_task": 22629.957912457914, "last_run_timestamp": "2026-10-01T11:20:35.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8881577165991904, "accuracy_std_dev": 0.008283430245979028, "run_count": 10, "total_questions": 1976, "avg_cost_per_task_usd": 0.015942041497975706, "avg_duration_per_task_ms": 164141.9762145749, "avg_output_tokens_per_task": 26128.10374493927, "last_run_timestamp": "2026-10-03T13:59:16.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.8880469999999999, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11352663888888888, "avg_duration_per_task_ms": 879783.9520202021, "avg_output_tokens_per_task": 45200.48484848485, "last_run_timestamp": "2026-10-03T10:42:46.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8880445319148936, "accuracy_std_dev": 0.009804096960454846, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.20574746403242147, "avg_duration_per_task_ms": 100391.19402228977, "avg_output_tokens_per_task": 8003.147416413374, "last_run_timestamp": "2026-10-03T09:42:47.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.887854866328257, "accuracy_std_dev": 0.036201773425286826, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.13917155047941343, "avg_duration_per_task_ms": 799755.2588832488, "avg_output_tokens_per_task": 34841.37958262832, "last_run_timestamp": "2026-10-03T10:47:57.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8877663333333334, "accuracy_std_dev": 0.005555502277522277, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03440870875420875, "avg_duration_per_task_ms": 50195.98148148148, "avg_output_tokens_per_task": 3309.2491582491584, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.887192300254453, "accuracy_std_dev": 0.021070431267668825, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.19019277692960135, "avg_duration_per_task_ms": 1311560.4325699746, "avg_output_tokens_per_task": 55844.176420695505, "last_run_timestamp": "2026-10-03T17:43:47.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8867842499999999, "accuracy_std_dev": 0.003834422340783562, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.027946079545454545, "avg_duration_per_task_ms": 154476.702020202, "avg_output_tokens_per_task": 23124.396464646463, "last_run_timestamp": "2026-10-03T10:53:39.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8867689502093997, "accuracy_std_dev": 0.008568113097385388, "run_count": 11, "total_questions": 2149, "avg_cost_per_task_usd": 0.09704398464402049, "avg_duration_per_task_ms": 550284.3764541647, "avg_output_tokens_per_task": 29436.060027919964, "last_run_timestamp": "2026-10-03T17:04:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.031905555555555555, "avg_duration_per_task_ms": 767151.1060606061, "avg_output_tokens_per_task": 36810.59595959596, "last_run_timestamp": "2026-10-03T10:37:19.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8855853621621622, "accuracy_std_dev": 0.025702889954244423, "run_count": 3, "total_questions": 555, "avg_cost_per_task_usd": 0.012490181981981983, "avg_duration_per_task_ms": 346127.65405405406, "avg_output_tokens_per_task": 22279.807207207206, "last_run_timestamp": "2026-10-03T10:21:30.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8855220000000001, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1415003787878788, "avg_duration_per_task_ms": 184472.2676767677, "avg_output_tokens_per_task": 47043.42424242424, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8853236853832442, "accuracy_std_dev": 0.023583528877870113, "run_count": 6, "total_questions": 1122, "avg_cost_per_task_usd": 0.1068478440285205, "avg_duration_per_task_ms": 683979.679144385, "avg_output_tokens_per_task": 29631.141711229946, "last_run_timestamp": "2026-10-05T21:52:29.584Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8853190481927712, "accuracy_std_dev": 0.005795847495189989, "run_count": 4, "total_questions": 747, "avg_cost_per_task_usd": 0.0097919718875502, "avg_duration_per_task_ms": 50542.31994645247, "avg_output_tokens_per_task": 7272.5394912985275, "last_run_timestamp": "2026-10-01T19:09:12.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8846155538461539, "accuracy_std_dev": 0.0024989999999999735, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.10645926410256411, "avg_duration_per_task_ms": 514880.74102564104, "avg_output_tokens_per_task": 34685.94102564103, "last_run_timestamp": "2026-10-03T17:17:05.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8845953462138955, "accuracy_std_dev": 0.012778996436692273, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.007367392271662762, "avg_duration_per_task_ms": 467182.3048399688, "avg_output_tokens_per_task": 26353.96487119438, "last_run_timestamp": "2026-10-03T09:22:28.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8845707161572053, "accuracy_std_dev": 0.02143221713073307, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.16085208820960697, "avg_duration_per_task_ms": 756655.7283842795, "avg_output_tokens_per_task": 40813.194759825325, "last_run_timestamp": "2026-09-16T00:37:42.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8843432, "accuracy_std_dev": 0.016512865255914843, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.11556713585858584, "avg_duration_per_task_ms": 607287.5232323232, "avg_output_tokens_per_task": 30218.88484848485, "last_run_timestamp": "2026-10-03T16:57:22.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.884269595505618, "accuracy_std_dev": 0.010583520080008851, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.018860816853932583, "avg_duration_per_task_ms": 74171.62584269662, "avg_output_tokens_per_task": 7862.741573033708, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.884259, "accuracy_std_dev": 0.003834285135458768, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028357917929292934, "avg_duration_per_task_ms": 90698.91287878787, "avg_output_tokens_per_task": 23491.551767676767, "last_run_timestamp": "2026-10-03T10:07:51.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8841843306451613, "accuracy_std_dev": 0.016935757620711884, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.04011120698924731, "avg_duration_per_task_ms": 259031.17204301077, "avg_output_tokens_per_task": 34712.397849462366, "last_run_timestamp": "2026-10-03T11:48:18.674Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8840485, "accuracy_std_dev": 0.014246189262395764, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.18107499936868682, "avg_duration_per_task_ms": 298705.5252525252, "avg_output_tokens_per_task": 45444.8851010101, "last_run_timestamp": "2026-10-03T10:13:34.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8839788953995158, "accuracy_std_dev": 0.013479642880336476, "run_count": 21, "total_questions": 4130, "avg_cost_per_task_usd": 0.19186280629539954, "avg_duration_per_task_ms": 375394.6331719128, "avg_output_tokens_per_task": 31206.65593220339, "last_run_timestamp": "2026-10-03T16:59:23.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8838383333333335, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13180945791245793, "avg_duration_per_task_ms": 248970.3754208754, "avg_output_tokens_per_task": 43817.097643097644, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8829965000000001, "accuracy_std_dev": 0.009135988652028878, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.02256884242424242, "avg_duration_per_task_ms": 361325.05656565656, "avg_output_tokens_per_task": 25803.312121212122, "last_run_timestamp": "2026-10-03T11:18:47.195Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.882652041571654, "accuracy_std_dev": 0.010061769676464596, "run_count": 30, "total_questions": 5701, "avg_cost_per_task_usd": 0.13625698438870373, "avg_duration_per_task_ms": 62308.83844939484, "avg_output_tokens_per_task": 4947.5786704087, "last_run_timestamp": "2026-10-03T09:33:58.131Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8823811431670283, "accuracy_std_dev": 0.01015848669631376, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.00803116413593637, "avg_duration_per_task_ms": 247598.04772234274, "avg_output_tokens_per_task": 28350.216919739698, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8822876450184503, "accuracy_std_dev": 0.011212925575387701, "run_count": 7, "total_questions": 1355, "avg_cost_per_task_usd": 0.07029816752767527, "avg_duration_per_task_ms": 312193.415498155, "avg_output_tokens_per_task": 19426.869372693727, "last_run_timestamp": "2026-10-03T18:59:42.075Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.882236963045085, "accuracy_std_dev": 0.03276554586072114, "run_count": 7, "total_questions": 1353, "avg_cost_per_task_usd": 0.006029221729490022, "avg_duration_per_task_ms": 323803.38359201775, "avg_output_tokens_per_task": 25311.461197339246, "last_run_timestamp": "2026-10-03T09:49:39.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8816051722365038, "accuracy_std_dev": 0.07456878886603124, "run_count": 13, "total_questions": 2334, "avg_cost_per_task_usd": 0.010089964010282775, "avg_duration_per_task_ms": 149337.25192802056, "avg_output_tokens_per_task": 23612.06083976007, "last_run_timestamp": "2026-10-03T10:12:41.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.013561239749947471, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005247140572390572, "avg_duration_per_task_ms": 250251.0606060606, "avg_output_tokens_per_task": 28487.505892255893, "last_run_timestamp": "2026-10-03T17:07:12.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8815301729019211, "accuracy_std_dev": 0.0059611119566738745, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.014240532861476238, "avg_duration_per_task_ms": 332365.4651162791, "avg_output_tokens_per_task": 26073.906976744187, "last_run_timestamp": "2026-10-03T10:46:54.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8813523333333333, "accuracy_std_dev": 0.012418655370047113, "run_count": 5, "total_questions": 774, "avg_cost_per_task_usd": 0.007396838501291989, "avg_duration_per_task_ms": 996564.0413436693, "avg_output_tokens_per_task": 29356.904392764856, "last_run_timestamp": "2026-10-03T09:53:30.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8813128999999998, "accuracy_std_dev": 0.018445584861695216, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.024727196464646464, "avg_duration_per_task_ms": 216333.7505050505, "avg_output_tokens_per_task": 25254.168181818182, "last_run_timestamp": "2026-10-03T09:13:55.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8810132745849297, "accuracy_std_dev": 0.029062612893329138, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.03828848659003832, "avg_duration_per_task_ms": 742925.4495530013, "avg_output_tokens_per_task": 25587.997445721583, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.880903752293578, "accuracy_std_dev": 0.010625848964200456, "run_count": 10, "total_questions": 1962, "avg_cost_per_task_usd": 0.005325050458715596, "avg_duration_per_task_ms": 400241.4791029562, "avg_output_tokens_per_task": 18619.075433231395, "last_run_timestamp": "2026-10-03T10:33:52.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.8808888960000001, "accuracy_std_dev": 0.009359000000000006, "run_count": 2, "total_questions": 375, "avg_cost_per_task_usd": 0.10911519200000001, "avg_duration_per_task_ms": 1097247.664, "avg_output_tokens_per_task": 43639.90933333334, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8804712499999999, "accuracy_std_dev": 0.006838655510222777, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028478232323232322, "avg_duration_per_task_ms": 124855.71338383839, "avg_output_tokens_per_task": 23539.155303030304, "last_run_timestamp": "2026-10-03T10:18:21.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.880471, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006361709595959596, "avg_duration_per_task_ms": 516019.3207070707, "avg_output_tokens_per_task": 34951.01515151515, "last_run_timestamp": "2026-10-03T11:41:19.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8796295, "accuracy_std_dev": 0.015267782951365261, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028045234848484847, "avg_duration_per_task_ms": 222260.06818181818, "avg_output_tokens_per_task": 27117.741161616163, "last_run_timestamp": "2026-10-03T09:35:14.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8794991295774648, "accuracy_std_dev": 0.01942029328961192, "run_count": 11, "total_questions": 2130, "avg_cost_per_task_usd": 0.12990812910798122, "avg_duration_per_task_ms": 63213.040375586854, "avg_output_tokens_per_task": 4970.991549295774, "last_run_timestamp": "2026-10-03T10:06:49.792Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8793958967310551, "accuracy_std_dev": 0.04684221744738129, "run_count": 7, "total_questions": 1346, "avg_cost_per_task_usd": 0.05736572734026745, "avg_duration_per_task_ms": 634171.9687964339, "avg_output_tokens_per_task": 19702.684249628528, "last_run_timestamp": "2026-10-03T17:12:28.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.879189, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.020723571428571428, "avg_duration_per_task_ms": 141095.54497354498, "avg_output_tokens_per_task": 17734.21164021164, "last_run_timestamp": "2026-09-15T00:33:26.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8790546552363299, "accuracy_std_dev": 0.030844451113161732, "run_count": 6, "total_questions": 1079, "avg_cost_per_task_usd": 0.1346450278035218, "avg_duration_per_task_ms": 284974.0537534754, "avg_output_tokens_per_task": 31126.302131603337, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "Deepseek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010643959595959596, "avg_duration_per_task_ms": 127627.76767676767, "avg_output_tokens_per_task": 14277.949494949495, "last_run_timestamp": "2026-10-08T13:29:57.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Mimo V2.6 Pro", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01249009595959596, "avg_duration_per_task_ms": 315799.5808080808, "avg_output_tokens_per_task": 14226.873737373737, "last_run_timestamp": "2026-10-08T01:55:44.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8787876666666666, "accuracy_std_dev": 0.005991714074916775, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007907055555555554, "avg_duration_per_task_ms": 79561.398989899, "avg_output_tokens_per_task": 7834.653198653199, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8787876363636363, "accuracy_std_dev": 0.006091232690182722, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.005402353535353536, "avg_duration_per_task_ms": 380915.31404958677, "avg_output_tokens_per_task": 29413.866391184572, "last_run_timestamp": "2026-10-03T09:51:11.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.8787874999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04341706060606061, "avg_duration_per_task_ms": 244250.4595959596, "avg_output_tokens_per_task": 37643.704545454544, "last_run_timestamp": "2026-10-03T10:51:03.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8772726000000001, "accuracy_std_dev": 0.03313293645362571, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.14829554040404042, "avg_duration_per_task_ms": 354670.0207070707, "avg_output_tokens_per_task": 34320.64595959596, "last_run_timestamp": "2026-10-03T17:43:07.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8770865174506829, "accuracy_std_dev": 0.007621402236465436, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.11726762670713202, "avg_duration_per_task_ms": 751432.9919069296, "avg_output_tokens_per_task": 30746.914516944868, "last_run_timestamp": "2026-10-03T16:47:49.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.876843640328698, "accuracy_std_dev": 0.014698550138959288, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.08638480594184576, "avg_duration_per_task_ms": 222833.85398230088, "avg_output_tokens_per_task": 21940.641592920354, "last_run_timestamp": "2026-10-03T09:40:59.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8761782480719793, "accuracy_std_dev": 0.00788648157846706, "run_count": 4, "total_questions": 778, "avg_cost_per_task_usd": 0.0930879910025707, "avg_duration_per_task_ms": 616690.5848329049, "avg_output_tokens_per_task": 27406.845758354757, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.8757659457720588, "accuracy_std_dev": 0.09251418420879137, "run_count": 11, "total_questions": 2176, "avg_cost_per_task_usd": 0.2399917977941176, "avg_duration_per_task_ms": 482095.9535845588, "avg_output_tokens_per_task": 59848.47564338235, "last_run_timestamp": "2026-10-03T10:18:13.963Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8757225447976879, "accuracy_std_dev": 0.004672056689969172, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.12312977673410404, "avg_duration_per_task_ms": 937524.2485549133, "avg_output_tokens_per_task": 43259.596098265894, "last_run_timestamp": "2026-10-03T17:24:45.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8757013333333332, "accuracy_std_dev": 0.025185257232930715, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09241545538720539, "avg_duration_per_task_ms": 486271.6456228956, "avg_output_tokens_per_task": 35368.11616161616, "last_run_timestamp": "2026-10-03T17:33:23.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.875421, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.04712759090909091, "avg_duration_per_task_ms": 154613.29292929292, "avg_output_tokens_per_task": 39139.681818181816, "last_run_timestamp": "2026-10-08T18:00:04.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.875421, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1538974696969697, "avg_duration_per_task_ms": 363407.6313131313, "avg_output_tokens_per_task": 34764.520202020205, "last_run_timestamp": "2026-07-18T08:26:40.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8754206666666666, "accuracy_std_dev": 0.00901385812081723, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.061844909090909096, "avg_duration_per_task_ms": 262173.4595959596, "avg_output_tokens_per_task": 22965.14478114478, "last_run_timestamp": "2026-09-26T09:19:06.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8753905728900255, "accuracy_std_dev": 0.012094409094012564, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.035846069053708435, "avg_duration_per_task_ms": 251997.9505541347, "avg_output_tokens_per_task": 29905.09377664109, "last_run_timestamp": "2026-10-03T12:29:29.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8749397142857143, "accuracy_std_dev": 0.007830033729434499, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.049728212842712845, "avg_duration_per_task_ms": 278314.67604617606, "avg_output_tokens_per_task": 21125.652958152958, "last_run_timestamp": "2026-10-03T17:26:10.527Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8748596666666666, "accuracy_std_dev": 0.03480145848157005, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10809361952861953, "avg_duration_per_task_ms": 516538.27384960715, "avg_output_tokens_per_task": 34956.31930415264, "last_run_timestamp": "2026-10-03T09:34:05.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8747592697495183, "accuracy_std_dev": 0.04128530357388087, "run_count": 4, "total_questions": 519, "avg_cost_per_task_usd": 0.012099612716763006, "avg_duration_per_task_ms": 465229.36030828516, "avg_output_tokens_per_task": 26222.763005780347, "last_run_timestamp": "2026-10-03T09:28:13.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8747029593082402, "accuracy_std_dev": 0.008749466763180506, "run_count": 5, "total_questions": 983, "avg_cost_per_task_usd": 0.1150072502543235, "avg_duration_per_task_ms": 564566.1881993896, "avg_output_tokens_per_task": 32828.28179043744, "last_run_timestamp": "2026-10-03T16:53:50.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8746725555555557, "accuracy_std_dev": 0.018426695846040054, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13061521717171717, "avg_duration_per_task_ms": 63177.591470258136, "avg_output_tokens_per_task": 5005.783950617284, "last_run_timestamp": "2026-10-03T09:54:43.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8744108, "accuracy_std_dev": 0.01990792291928014, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.008221054545454545, "avg_duration_per_task_ms": 359745.57575757575, "avg_output_tokens_per_task": 29012.32424242424, "last_run_timestamp": "2026-10-03T17:03:58.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8743913028237584, "accuracy_std_dev": 0.02063518118543075, "run_count": 6, "total_questions": 1027, "avg_cost_per_task_usd": 0.1379305988315482, "avg_duration_per_task_ms": 596483.0662122688, "avg_output_tokens_per_task": 50109.39727361246, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8743494545454547, "accuracy_std_dev": 0.0150964078771537, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1106280968778696, "avg_duration_per_task_ms": 234771.5486685032, "avg_output_tokens_per_task": 31461.72359963269, "last_run_timestamp": "2026-10-03T16:40:49.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8742763324913168, "accuracy_std_dev": 0.03081876645203374, "run_count": 16, "total_questions": 3167, "avg_cost_per_task_usd": 0.14868513672245026, "avg_duration_per_task_ms": 406084.6277233975, "avg_output_tokens_per_task": 11240.50994632144, "last_run_timestamp": "2026-10-05T20:36:24.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8742182857142856, "accuracy_std_dev": 0.005225109204991012, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07788331168831168, "avg_duration_per_task_ms": 336872.55194805196, "avg_output_tokens_per_task": 22144.56132756133, "last_run_timestamp": "2026-10-03T19:00:32.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008290222222222223, "avg_duration_per_task_ms": 529447, "avg_output_tokens_per_task": 29241.171717171717, "last_run_timestamp": "2026-07-18T08:31:23.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8736602893401014, "accuracy_std_dev": 0.018240759785588822, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.033190517766497465, "avg_duration_per_task_ms": 731117.2707275804, "avg_output_tokens_per_task": 38434.73942470389, "last_run_timestamp": "2026-10-03T11:12:58.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.873140968503937, "accuracy_std_dev": 0.008141500000000024, "run_count": 2, "total_questions": 381, "avg_cost_per_task_usd": 0.10504513910761155, "avg_duration_per_task_ms": 1003143.2047244095, "avg_output_tokens_per_task": 42008.77427821522, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8727401757049892, "accuracy_std_dev": 0.012263522167761748, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.00852685032537961, "avg_duration_per_task_ms": 434756.32754880696, "avg_output_tokens_per_task": 21739.99276934201, "last_run_timestamp": "2026-10-03T10:06:09.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8727272475524476, "accuracy_std_dev": 0.029233686851601524, "run_count": 8, "total_questions": 1430, "avg_cost_per_task_usd": 0.09700304615384615, "avg_duration_per_task_ms": 194874.37272727274, "avg_output_tokens_per_task": 26397.99300699301, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8726482966714906, "accuracy_std_dev": 0.007320320889029898, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.0685544536903039, "avg_duration_per_task_ms": 284232.5897250362, "avg_output_tokens_per_task": 11158.96454413893, "last_run_timestamp": "2026-10-08T22:30:47.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8724745, "accuracy_std_dev": 0.008858340547190563, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008325782828282828, "avg_duration_per_task_ms": 77307.88005050505, "avg_output_tokens_per_task": 7842.704545454545, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8723343333333334, "accuracy_std_dev": 0.015226503403240287, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.2220637457912458, "avg_duration_per_task_ms": 974700.601010101, "avg_output_tokens_per_task": 55783.88468013468, "last_run_timestamp": "2026-10-03T16:37:45.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8722849626411816, "accuracy_std_dev": 0.03667207040635816, "run_count": 6, "total_questions": 1151, "avg_cost_per_task_usd": 0.01310653171155517, "avg_duration_per_task_ms": 596394.4969591659, "avg_output_tokens_per_task": 34928.014769765425, "last_run_timestamp": "2026-10-03T09:08:44.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8722010494296577, "accuracy_std_dev": 0.0020781669055925365, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.045619747782002534, "avg_duration_per_task_ms": 189717.51711026617, "avg_output_tokens_per_task": 39672.14828897338, "last_run_timestamp": "2026-10-03T10:44:50.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8720540000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004176666666666667, "avg_duration_per_task_ms": 40447.64646464647, "avg_output_tokens_per_task": 8116.434343434344, "last_run_timestamp": "2026-10-08T06:00:12.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8715861506361324, "accuracy_std_dev": 0.009632287238761115, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.008127365394402036, "avg_duration_per_task_ms": 324970.47837150126, "avg_output_tokens_per_task": 28742.415267175573, "last_run_timestamp": "2026-10-03T09:52:27.516Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8714926666666667, "accuracy_std_dev": 0.0028614651647170092, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004327225589225589, "avg_duration_per_task_ms": 40230.3164983165, "avg_output_tokens_per_task": 7890.072390572391, "last_run_timestamp": "2026-10-08T06:00:12.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8714225, "accuracy_std_dev": 0.004885840587862051, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06337124747474747, "avg_duration_per_task_ms": 284598.4861111111, "avg_output_tokens_per_task": 21034.015151515152, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8710353744745891, "accuracy_std_dev": 0.16260220955169502, "run_count": 15, "total_questions": 2617, "avg_cost_per_task_usd": 0.142784888803974, "avg_duration_per_task_ms": 265667.35957202903, "avg_output_tokens_per_task": 10833.450133740926, "last_run_timestamp": "2026-10-03T10:04:25.516Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8708512857142857, "accuracy_std_dev": 0.01450202738254489, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026409178932178933, "avg_duration_per_task_ms": 705209.8095238095, "avg_output_tokens_per_task": 34652.24603174603, "last_run_timestamp": "2026-10-03T16:57:03.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.8705104999999999, "accuracy_std_dev": 0.07295362229012346, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12380340740740742, "avg_duration_per_task_ms": 1287771.0867003368, "avg_output_tokens_per_task": 54917.97643097643, "last_run_timestamp": "2026-10-03T17:53:34.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8699495000000002, "accuracy_std_dev": 0.018722910197135487, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.030484161616161617, "avg_duration_per_task_ms": 134273.03535353535, "avg_output_tokens_per_task": 21527.786616161615, "last_run_timestamp": "2026-10-03T12:27:06.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8698091666666667, "accuracy_std_dev": 0.0056397381268006655, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008173530303030302, "avg_duration_per_task_ms": 420480.94865319866, "avg_output_tokens_per_task": 28865.37626262626, "last_run_timestamp": "2026-10-03T16:58:32.515Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.010411879221190886, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07784478367003367, "avg_duration_per_task_ms": 107880.84595959596, "avg_output_tokens_per_task": 12619.909932659933, "last_run_timestamp": "2026-10-03T10:07:47.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8693955131578947, "accuracy_std_dev": 0.019648291273268125, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.07663164254385965, "avg_duration_per_task_ms": 433978.8881578947, "avg_output_tokens_per_task": 23705.32821637427, "last_run_timestamp": "2026-10-08T23:14:17.356Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8692988181818181, "accuracy_std_dev": 0.01574807678773497, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.008268114784205692, "avg_duration_per_task_ms": 218163.90817263545, "avg_output_tokens_per_task": 29305.03948576676, "last_run_timestamp": "2026-10-03T10:06:44.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.869276, "accuracy_std_dev": 0.008908612349855623, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.006512984848484849, "avg_duration_per_task_ms": 255242.29242424242, "avg_output_tokens_per_task": 18235.289393939394, "last_run_timestamp": "2026-10-03T09:02:06.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8692479999999999, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22171807575757574, "avg_duration_per_task_ms": 260359.50673400675, "avg_output_tokens_per_task": 22076.11111111111, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8692478333333333, "accuracy_std_dev": 0.012955574635611068, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.08235681228956229, "avg_duration_per_task_ms": 202641.79545454544, "avg_output_tokens_per_task": 13365.021885521886, "last_run_timestamp": "2026-10-03T09:43:24.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8691446346328195, "accuracy_std_dev": 0.025234963139792104, "run_count": 6, "total_questions": 1103, "avg_cost_per_task_usd": 0.01014118223028105, "avg_duration_per_task_ms": 374535.42611060746, "avg_output_tokens_per_task": 21668.27742520399, "last_run_timestamp": "2026-10-03T09:34:29.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8689928181818182, "accuracy_std_dev": 0.05184823150279035, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.07711455371900827, "avg_duration_per_task_ms": 533320.4696969697, "avg_output_tokens_per_task": 38582.5886134068, "last_run_timestamp": "2026-10-08T22:52:10.694Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13888562626262627, "avg_duration_per_task_ms": 1028019.2020202021, "avg_output_tokens_per_task": 46184.969696969696, "last_run_timestamp": "2026-07-18T16:16:27.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.868234624430956, "accuracy_std_dev": 0.009898395121653556, "run_count": 8, "total_questions": 1318, "avg_cost_per_task_usd": 0.05128331487101669, "avg_duration_per_task_ms": 415827.8520485584, "avg_output_tokens_per_task": 17002.027314112293, "last_run_timestamp": "2026-10-08T23:30:45.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8679651428571429, "accuracy_std_dev": 0.004916957259716384, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07019697691197692, "avg_duration_per_task_ms": 226467.18253968254, "avg_output_tokens_per_task": 11432.188311688311, "last_run_timestamp": "2026-10-03T09:14:11.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8678913937007873, "accuracy_std_dev": 0.012999770226334775, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.008103982283464567, "avg_duration_per_task_ms": 243353.82217847768, "avg_output_tokens_per_task": 29838.47900262467, "last_run_timestamp": "2026-10-09T21:00:06.195Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8678219449541285, "accuracy_std_dev": 0.011378331853132084, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.1803859877675841, "avg_duration_per_task_ms": 784706.3690112131, "avg_output_tokens_per_task": 52291.86748216106, "last_run_timestamp": "2026-10-03T17:02:58.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.8677916075949366, "accuracy_std_dev": 0.019070276800560625, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.057791014345991557, "avg_duration_per_task_ms": 510760.66835443035, "avg_output_tokens_per_task": 33916.00421940928, "last_run_timestamp": "2026-10-03T17:23:31.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.867634625, "accuracy_std_dev": 0.007431114989311851, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025694838383838382, "avg_duration_per_task_ms": 248942.36553030304, "avg_output_tokens_per_task": 21167.008207070707, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8675644999999997, "accuracy_std_dev": 0.008112634749779732, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07749805892255893, "avg_duration_per_task_ms": 84869.17171717172, "avg_output_tokens_per_task": 12670.825757575758, "last_run_timestamp": "2026-10-03T09:34:51.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8670031666666665, "accuracy_std_dev": 0.018364664062783427, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1649758611111111, "avg_duration_per_task_ms": 1021779.7348484849, "avg_output_tokens_per_task": 53789.280303030304, "last_run_timestamp": "2026-10-03T17:43:48.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8670031666666665, "accuracy_std_dev": 0.02310358935617773, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.19514486279461277, "avg_duration_per_task_ms": 1047935.0816498316, "avg_output_tokens_per_task": 57929.56986531986, "last_run_timestamp": "2026-10-03T17:03:53.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.867003, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11555696464646466, "avg_duration_per_task_ms": 292702.6111111111, "avg_output_tokens_per_task": 29334.075757575756, "last_run_timestamp": "2026-07-18T08:27:05.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.866915421641791, "accuracy_std_dev": 0.02922832992755772, "run_count": 12, "total_questions": 2144, "avg_cost_per_task_usd": 0.010834175839552238, "avg_duration_per_task_ms": 328640.2140858209, "avg_output_tokens_per_task": 19735.994869402984, "last_run_timestamp": "2026-10-03T09:27:10.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.866685276559865, "accuracy_std_dev": 0.03454921866619338, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.11449315514333895, "avg_duration_per_task_ms": 640608.7127599776, "avg_output_tokens_per_task": 38548.483417650365, "last_run_timestamp": "2026-10-03T16:56:36.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.8661615, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.030201459595959598, "avg_duration_per_task_ms": 496509.8611111111, "avg_output_tokens_per_task": 34831.292929292926, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8660412857142855, "accuracy_std_dev": 0.006542422852086991, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0036472380952380944, "avg_duration_per_task_ms": 423809.5887445887, "avg_output_tokens_per_task": 29733.171717171717, "last_run_timestamp": "2026-10-03T17:32:07.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8658809999999999, "accuracy_std_dev": 0.011527973831800053, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004285917508417508, "avg_duration_per_task_ms": 35717.200336700334, "avg_output_tokens_per_task": 7821.843434343435, "last_run_timestamp": "2026-10-08T06:00:12.146Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8658007142857141, "accuracy_std_dev": 0.009679853093842238, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.06976018037518038, "avg_duration_per_task_ms": 223743.95815295816, "avg_output_tokens_per_task": 11357.300144300145, "last_run_timestamp": "2026-10-03T09:30:30.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.865689794871795, "accuracy_std_dev": 0.004877364201651907, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.005803672161172161, "avg_duration_per_task_ms": 153675.5, "avg_output_tokens_per_task": 20639.26923076923, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.86532, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004150199494949496, "avg_duration_per_task_ms": 35865.53535353536, "avg_output_tokens_per_task": 7670.747474747474, "last_run_timestamp": "2026-10-08T06:00:12.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8651969388477632, "accuracy_std_dev": 0.0392226856144323, "run_count": 16, "total_questions": 3107, "avg_cost_per_task_usd": 0.004922328290955906, "avg_duration_per_task_ms": 103773.42162858062, "avg_output_tokens_per_task": 18562.228194399744, "last_run_timestamp": "2026-10-03T10:10:22.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8649528761766837, "accuracy_std_dev": 0.01859749373173817, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.0065336495293265755, "avg_duration_per_task_ms": 348157.4265025344, "avg_output_tokens_per_task": 21785.366401158582, "last_run_timestamp": "2026-10-03T10:20:35.518Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8649467344461305, "accuracy_std_dev": 0.10260638821189447, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.026698589782498738, "avg_duration_per_task_ms": 213357.39251390996, "avg_output_tokens_per_task": 23147.549317147194, "last_run_timestamp": "2026-10-03T17:06:35.452Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.864636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.07161002030456853, "avg_duration_per_task_ms": 868248.1269035533, "avg_output_tokens_per_task": 28363.54314720812, "last_run_timestamp": "2026-10-03T14:58:52.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8645714444444443, "accuracy_std_dev": 0.01709693694535898, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.08812040179573512, "avg_duration_per_task_ms": 199235.38664421998, "avg_output_tokens_per_task": 24344.7822671156, "last_run_timestamp": "2026-10-03T16:06:47.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8643883344182262, "accuracy_std_dev": 0.01774344793127301, "run_count": 7, "total_questions": 1229, "avg_cost_per_task_usd": 0.0051325646867371845, "avg_duration_per_task_ms": 651023.4035801465, "avg_output_tokens_per_task": 28234.647681041497, "last_run_timestamp": "2026-10-08T23:29:09.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8642566289198608, "accuracy_std_dev": 0.016122613680445513, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.08601719425087108, "avg_duration_per_task_ms": 391195.4764808362, "avg_output_tokens_per_task": 24802.604529616725, "last_run_timestamp": "2026-10-03T09:33:56.518Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8642559853907962, "accuracy_std_dev": 0.01553435314759793, "run_count": 7, "total_questions": 1369, "avg_cost_per_task_usd": 0.07498870489408328, "avg_duration_per_task_ms": 273379.0087655223, "avg_output_tokens_per_task": 20706.99488677867, "last_run_timestamp": "2026-10-03T17:17:14.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8641975, "accuracy_std_dev": 0.020610892069890297, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.16472780976430976, "avg_duration_per_task_ms": 1088563.0942760943, "avg_output_tokens_per_task": 59591.08249158249, "last_run_timestamp": "2026-10-03T17:43:01.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8641171428571428, "accuracy_std_dev": 0.010825054462793655, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008647630591630592, "avg_duration_per_task_ms": 270553.19336219336, "avg_output_tokens_per_task": 30573.133477633477, "last_run_timestamp": "2026-10-03T17:35:32.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.864098116850127, "accuracy_std_dev": 0.012040853347389158, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.06984978577476715, "avg_duration_per_task_ms": 203374.89754445385, "avg_output_tokens_per_task": 24608.592718035565, "last_run_timestamp": "2026-10-03T09:19:06.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8639166666666666, "accuracy_std_dev": 0.022215769209180104, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1178505563973064, "avg_duration_per_task_ms": 576577.5420875421, "avg_output_tokens_per_task": 35099.228114478115, "last_run_timestamp": "2026-10-03T17:19:05.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003967691919191919, "avg_duration_per_task_ms": 39901.43939393939, "avg_output_tokens_per_task": 7695.671717171717, "last_run_timestamp": "2026-10-08T06:00:12.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008496712121212122, "avg_duration_per_task_ms": 739364.4242424242, "avg_output_tokens_per_task": 30053.868686868685, "last_run_timestamp": "2026-07-18T08:30:06.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.863636, "accuracy_std_dev": 0.007273551310513057, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020974939393939393, "avg_duration_per_task_ms": 363755.6632996633, "avg_output_tokens_per_task": 34831.013468013465, "last_run_timestamp": "2026-10-03T11:53:46.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.863636, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22125699158249157, "avg_duration_per_task_ms": 259075.24915824915, "avg_output_tokens_per_task": 22030.511784511786, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8636106825127334, "accuracy_std_dev": 0.03982580471794763, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.08851625127334464, "avg_duration_per_task_ms": 511731.79343520093, "avg_output_tokens_per_task": 22926.203735144314, "last_run_timestamp": "2026-10-03T17:07:22.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.8629148571428572, "accuracy_std_dev": 0.017903828987346902, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07367307792207792, "avg_duration_per_task_ms": 277976.63924963924, "avg_output_tokens_per_task": 20972.66738816739, "last_run_timestamp": "2026-10-03T17:13:56.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.8625952061068702, "accuracy_std_dev": 0.005736999999999992, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.043527526717557254, "avg_duration_per_task_ms": 365946.2213740458, "avg_output_tokens_per_task": 24956.239185750637, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8625138888888889, "accuracy_std_dev": 0.02716878807440808, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00797921548821549, "avg_duration_per_task_ms": 504816.8608305275, "avg_output_tokens_per_task": 25080.10830527497, "last_run_timestamp": "2026-10-03T10:11:44.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8625137777777779, "accuracy_std_dev": 0.00880146315838422, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8542647558922559, "avg_duration_per_task_ms": 483293.1627384961, "avg_output_tokens_per_task": 33979.86139169472, "last_run_timestamp": "2026-10-03T09:32:40.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8625002076923076, "accuracy_std_dev": 0.01982845351172571, "run_count": 3, "total_questions": 520, "avg_cost_per_task_usd": 0.017099969230769233, "avg_duration_per_task_ms": 517523.88653846155, "avg_output_tokens_per_task": 25671.957692307693, "last_run_timestamp": "2026-10-09T03:54:06.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8622331666666665, "accuracy_std_dev": 0.014944828898503846, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.047331268518518516, "avg_duration_per_task_ms": 475487.20454545453, "avg_output_tokens_per_task": 33266.92003367003, "last_run_timestamp": "2026-10-03T17:33:25.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8618355304347826, "accuracy_std_dev": 0.022183205439048254, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.07637872463768115, "avg_duration_per_task_ms": 413540.6456521739, "avg_output_tokens_per_task": 32373.957971014494, "last_run_timestamp": "2026-10-10T00:00:05.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8617657777777777, "accuracy_std_dev": 0.009167555942050057, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8453705723905725, "avg_duration_per_task_ms": 415699.1722783389, "avg_output_tokens_per_task": 33623.734006734005, "last_run_timestamp": "2026-10-03T09:39:37.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8617122857142857, "accuracy_std_dev": 0.015778023212360787, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04849973448773449, "avg_duration_per_task_ms": 719703.6161616162, "avg_output_tokens_per_task": 33196.97546897547, "last_run_timestamp": "2026-10-03T17:03:53.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8616955969987624, "accuracy_std_dev": 0.021294415333698184, "run_count": 33, "total_questions": 6464, "avg_cost_per_task_usd": 0.2610163531868812, "avg_duration_per_task_ms": 338765.33369430696, "avg_output_tokens_per_task": 47192.89201732673, "last_run_timestamp": "2026-10-03T17:31:59.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8615786666666667, "accuracy_std_dev": 0.009382650075775203, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8094348821548821, "avg_duration_per_task_ms": 461988.8821548821, "avg_output_tokens_per_task": 32205.768237934906, "last_run_timestamp": "2026-10-03T10:13:16.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8614377764705882, "accuracy_std_dev": 0.012431326538225896, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.051462403921568624, "avg_duration_per_task_ms": 413319.64183006535, "avg_output_tokens_per_task": 26795.732026143793, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.015601910243620808, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.19432850925925926, "avg_duration_per_task_ms": 386703.5462962963, "avg_output_tokens_per_task": 56841.80892255892, "last_run_timestamp": "2026-10-03T16:58:25.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8609939949452401, "accuracy_std_dev": 0.034291003424934796, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.12772796882898063, "avg_duration_per_task_ms": 665372.3294018534, "avg_output_tokens_per_task": 42817.63689974726, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8609694285714284, "accuracy_std_dev": 0.023293052022008633, "run_count": 4, "total_questions": 784, "avg_cost_per_task_usd": 0.10234140178571428, "avg_duration_per_task_ms": 463624.993622449, "avg_output_tokens_per_task": 30066.539540816328, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8608923543307087, "accuracy_std_dev": 0.017439499999999997, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.003837814960629921, "avg_duration_per_task_ms": 298144.1771653543, "avg_output_tokens_per_task": 11523.807086614173, "last_run_timestamp": "2026-10-03T17:04:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8607369444444446, "accuracy_std_dev": 0.008186229884332315, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.8829101122334456, "avg_duration_per_task_ms": 416465.39506172837, "avg_output_tokens_per_task": 33444.42985409652, "last_run_timestamp": "2026-10-03T09:27:25.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8606576691394658, "accuracy_std_dev": 0.011001380598888357, "run_count": 24, "total_questions": 4044, "avg_cost_per_task_usd": 0.16706140232443123, "avg_duration_per_task_ms": 253403.07789317507, "avg_output_tokens_per_task": 30580.46612265084, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.8602695000000001, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07283564141414141, "avg_duration_per_task_ms": 493206.3787878788, "avg_output_tokens_per_task": 56881.042929292926, "last_run_timestamp": "2026-10-03T17:41:27.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8602693333333333, "accuracy_std_dev": 0.008473596416057473, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0050054402356902354, "avg_duration_per_task_ms": 240643.9755892256, "avg_output_tokens_per_task": 28765.88973063973, "last_run_timestamp": "2026-10-03T16:57:52.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.8602692222222225, "accuracy_std_dev": 0.01392791126381983, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.7468858333333334, "avg_duration_per_task_ms": 493290.462962963, "avg_output_tokens_per_task": 29714.63973063973, "last_run_timestamp": "2026-10-03T09:34:14.060Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.8599289999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 188, "avg_cost_per_task_usd": 0.023036143617021276, "avg_duration_per_task_ms": 1599986.2287234042, "avg_output_tokens_per_task": 29232.622340425532, "last_run_timestamp": "2026-07-18T08:57:54.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8598355714285714, "accuracy_std_dev": 0.011856733262862344, "run_count": 6, "total_questions": 1176, "avg_cost_per_task_usd": 0.14287056972789117, "avg_duration_per_task_ms": 268723.07568027213, "avg_output_tokens_per_task": 28901.64880952381, "last_run_timestamp": "2026-10-03T09:42:23.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8596616913043478, "accuracy_std_dev": 0.02724773313095075, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.10258134202898551, "avg_duration_per_task_ms": 343569.69492753624, "avg_output_tokens_per_task": 23077.884057971016, "last_run_timestamp": "2026-10-03T09:34:01.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8592168750000002, "accuracy_std_dev": 0.04194354929377548, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.17683030808080807, "avg_duration_per_task_ms": 45565.645833333336, "avg_output_tokens_per_task": 3337.2266414141413, "last_run_timestamp": "2026-10-03T09:18:12.856Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8590668571428572, "accuracy_std_dev": 0.0075671013591461535, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.06653240404040404, "avg_duration_per_task_ms": 215829.40548340548, "avg_output_tokens_per_task": 10820.84126984127, "last_run_timestamp": "2026-10-03T10:11:33.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2511099898989899, "avg_duration_per_task_ms": 290522.3080808081, "avg_output_tokens_per_task": 62590.06565656565, "last_run_timestamp": "2026-07-18T08:26:40.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8578204545454546, "accuracy_std_dev": 0.010161020030789983, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8425653397612488, "avg_duration_per_task_ms": 451495.49678604223, "avg_output_tokens_per_task": 33526.43617998163, "last_run_timestamp": "2026-10-03T10:12:25.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8578001999999999, "accuracy_std_dev": 0.012246153786393496, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.11574598114478116, "avg_duration_per_task_ms": 406292.1531986532, "avg_output_tokens_per_task": 41531.61616161616, "last_run_timestamp": "2026-10-03T17:10:11.381Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8576669736716133, "accuracy_std_dev": 0.03054982749271608, "run_count": 11, "total_questions": 2089, "avg_cost_per_task_usd": 0.025787480612733366, "avg_duration_per_task_ms": 476213.8056486357, "avg_output_tokens_per_task": 24583.015797032072, "last_run_timestamp": "2026-10-03T09:49:53.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8576504444444445, "accuracy_std_dev": 0.017188931406971766, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09760986363636363, "avg_duration_per_task_ms": 514558.1818181818, "avg_output_tokens_per_task": 26225.028058361393, "last_run_timestamp": "2026-10-03T16:43:18.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8573280311284046, "accuracy_std_dev": 0.008776662705721345, "run_count": 4, "total_questions": 771, "avg_cost_per_task_usd": 0.11767329182879378, "avg_duration_per_task_ms": 355412.9636835279, "avg_output_tokens_per_task": 29743.00907911803, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8572655795356836, "accuracy_std_dev": 0.023144525715238454, "run_count": 6, "total_questions": 1163, "avg_cost_per_task_usd": 0.02012267153912296, "avg_duration_per_task_ms": 381128.8787618229, "avg_output_tokens_per_task": 23297.414445399827, "last_run_timestamp": "2026-10-04T03:08:02.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8569023333333334, "accuracy_std_dev": 0.016379967131292483, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01228687542087542, "avg_duration_per_task_ms": 639472.5816498316, "avg_output_tokens_per_task": 33004.78787878788, "last_run_timestamp": "2026-10-03T09:19:24.461Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "WandB", "accuracy": 0.8569019999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006055888888888888, "avg_duration_per_task_ms": 490894.14646464644, "avg_output_tokens_per_task": 17030.60606060606, "last_run_timestamp": "2026-07-18T08:04:02.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8563952803943043, "accuracy_std_dev": 0.008474177580046515, "run_count": 14, "total_questions": 2739, "avg_cost_per_task_usd": 0.06252096166484118, "avg_duration_per_task_ms": 600270.9682365827, "avg_output_tokens_per_task": 34408.29098211026, "last_run_timestamp": "2026-10-03T17:02:38.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.8560605, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02547615909090909, "avg_duration_per_task_ms": 266390.93434343435, "avg_output_tokens_per_task": 21055.325757575756, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8557689653846154, "accuracy_std_dev": 0.016989351495046, "run_count": 3, "total_questions": 520, "avg_cost_per_task_usd": 0.06363424807692308, "avg_duration_per_task_ms": 459711.01153846155, "avg_output_tokens_per_task": 23017.521153846155, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.855219, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.19856715151515153, "avg_duration_per_task_ms": 638184.691919192, "avg_output_tokens_per_task": 58287.33838383838, "last_run_timestamp": "2026-07-18T08:26:48.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8550479847715735, "accuracy_std_dev": 0.00436888279642389, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.24340484940778345, "avg_duration_per_task_ms": 425015.06429780036, "avg_output_tokens_per_task": 54319.2165820643, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8550316666666666, "accuracy_std_dev": 0.01908702254878369, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8377942901234569, "avg_duration_per_task_ms": 451775.2300785634, "avg_output_tokens_per_task": 33341.289001122335, "last_run_timestamp": "2026-10-03T09:45:17.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8549221865284975, "accuracy_std_dev": 0.006241046279804878, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.02481670293609672, "avg_duration_per_task_ms": 142245.9913644214, "avg_output_tokens_per_task": 37982.34369602763, "last_run_timestamp": "2026-10-03T10:08:27.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8548752091836734, "accuracy_std_dev": 0.01338517551144632, "run_count": 6, "total_questions": 1176, "avg_cost_per_task_usd": 0.1090017406462585, "avg_duration_per_task_ms": 191170.07142857142, "avg_output_tokens_per_task": 28383.128401360544, "last_run_timestamp": "2026-10-03T09:07:14.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8548644736842105, "accuracy_std_dev": 0.024533883813741147, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.15887166959064328, "avg_duration_per_task_ms": 152796.97049441785, "avg_output_tokens_per_task": 15519.810738968634, "last_run_timestamp": "2026-10-03T09:27:53.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.854800172815534, "accuracy_std_dev": 0.012641847777693532, "run_count": 8, "total_questions": 1545, "avg_cost_per_task_usd": 0.1002615708737864, "avg_duration_per_task_ms": 668327.793527508, "avg_output_tokens_per_task": 36663.679611650485, "last_run_timestamp": "2026-09-19T08:55:10.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8547135999999999, "accuracy_std_dev": 0.03570492302526363, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1789077186868687, "avg_duration_per_task_ms": 44801.785858585856, "avg_output_tokens_per_task": 3372.5843434343433, "last_run_timestamp": "2026-10-03T09:14:11.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.854592, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.08009020408163266, "avg_duration_per_task_ms": 1565820.9795918367, "avg_output_tokens_per_task": 40916.48469387755, "last_run_timestamp": "2026-07-18T16:22:39.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8545491290566037, "accuracy_std_dev": 0.016485433040158266, "run_count": 21, "total_questions": 3975, "avg_cost_per_task_usd": 0.15811767823899373, "avg_duration_per_task_ms": 199794.28352201259, "avg_output_tokens_per_task": 28179.00402515723, "last_run_timestamp": "2026-10-03T09:59:27.517Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8545453999999999, "accuracy_std_dev": 0.04176428250119952, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.17536612575757574, "avg_duration_per_task_ms": 43901.6994949495, "avg_output_tokens_per_task": 3302.6388888888887, "last_run_timestamp": "2026-10-03T09:19:04.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8545453999999999, "accuracy_std_dev": 0.008010288199559339, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006016572727272727, "avg_duration_per_task_ms": 589898.7232323232, "avg_output_tokens_per_task": 28997.2, "last_run_timestamp": "2026-10-03T17:45:03.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8542019060913706, "accuracy_std_dev": 0.006301576321753829, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.05433487690355329, "avg_duration_per_task_ms": 423724.30245346867, "avg_output_tokens_per_task": 27804.93189509306, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8540964999999999, "accuracy_std_dev": 0.014793979437032262, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09252428872053871, "avg_duration_per_task_ms": 604262.7205387205, "avg_output_tokens_per_task": 27745.595117845118, "last_run_timestamp": "2026-10-03T09:59:46.432Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8537222222222223, "accuracy_std_dev": 0.017912548902175803, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.7545602300785634, "avg_duration_per_task_ms": 501662.9904601571, "avg_output_tokens_per_task": 30067.62962962963, "last_run_timestamp": "2026-10-03T09:46:07.607Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8535444426523295, "accuracy_std_dev": 0.03729714924788522, "run_count": 29, "total_questions": 5022, "avg_cost_per_task_usd": 0.012806513739546001, "avg_duration_per_task_ms": 408091.7891278375, "avg_output_tokens_per_task": 16584.42453205894, "last_run_timestamp": "2026-10-03T10:45:26.547Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8535352222222222, "accuracy_std_dev": 0.01802737382598276, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.8012126506734006, "avg_duration_per_task_ms": 478739.47053872055, "avg_output_tokens_per_task": 30372.589225589225, "last_run_timestamp": "2026-10-03T09:41:04.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8533247499999999, "accuracy_std_dev": 0.03034305490861954, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.8320753535353536, "avg_duration_per_task_ms": 427272.283459596, "avg_output_tokens_per_task": 33208.628787878784, "last_run_timestamp": "2026-10-03T09:44:19.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8531986, "accuracy_std_dev": 0.030700851614246794, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.12264740303030303, "avg_duration_per_task_ms": 564764.8696969697, "avg_output_tokens_per_task": 32450.073737373736, "last_run_timestamp": "2026-10-03T16:54:37.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8531398217391304, "accuracy_std_dev": 0.028934812364545037, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.012572355072463768, "avg_duration_per_task_ms": 541534.9311594203, "avg_output_tokens_per_task": 33634.00579710145, "last_run_timestamp": "2026-10-03T09:08:08.594Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.853024801369863, "accuracy_std_dev": 0.02604945551361019, "run_count": 6, "total_questions": 1168, "avg_cost_per_task_usd": 0.13878068065068494, "avg_duration_per_task_ms": 378053.25171232875, "avg_output_tokens_per_task": 31592.83133561644, "last_run_timestamp": "2026-10-03T09:40:06.763Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8528984782608696, "accuracy_std_dev": 0.014334577107719047, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.018874540579710147, "avg_duration_per_task_ms": 358751.04855072463, "avg_output_tokens_per_task": 37538.637681159424, "last_run_timestamp": "2026-10-03T10:13:52.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8527699090909091, "accuracy_std_dev": 0.015435615118376213, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7526788682277317, "avg_duration_per_task_ms": 500624.6831955923, "avg_output_tokens_per_task": 30015.131313131315, "last_run_timestamp": "2026-10-03T09:32:02.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8522125357142856, "accuracy_std_dev": 0.025081070583327737, "run_count": 28, "total_questions": 5544, "avg_cost_per_task_usd": 0.1703613856421356, "avg_duration_per_task_ms": 157589.71464646465, "avg_output_tokens_per_task": 15942.862554112555, "last_run_timestamp": "2026-10-03T09:43:58.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.851932886214442, "accuracy_std_dev": 0.02400418545476889, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.04229712910284464, "avg_duration_per_task_ms": 657747.1064916119, "avg_output_tokens_per_task": 35368.60831509847, "last_run_timestamp": "2026-10-03T16:51:59.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 144, "avg_cost_per_task_usd": 0.15792222916666668, "avg_duration_per_task_ms": 798429.2847222222, "avg_output_tokens_per_task": 42935.319444444445, "last_run_timestamp": "2026-07-18T08:29:37.313Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8518308614084509, "accuracy_std_dev": 0.02345464140331345, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.006422030422535212, "avg_duration_per_task_ms": 229362.44, "avg_output_tokens_per_task": 12608.14704225352, "last_run_timestamp": "2026-10-03T13:30:57.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.85145952259887, "accuracy_std_dev": 0.01855480961260448, "run_count": 5, "total_questions": 708, "avg_cost_per_task_usd": 0.003704364406779661, "avg_duration_per_task_ms": 411396.5324858757, "avg_output_tokens_per_task": 7770.766949152542, "last_run_timestamp": "2026-10-03T09:41:45.583Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8512626000000001, "accuracy_std_dev": 0.01008732084549707, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.040042293434343436, "avg_duration_per_task_ms": 396466.652020202, "avg_output_tokens_per_task": 40285.31212121212, "last_run_timestamp": "2026-10-03T19:06:53.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8508898571428573, "accuracy_std_dev": 0.059458570338462385, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1082869906204906, "avg_duration_per_task_ms": 468673.2698412698, "avg_output_tokens_per_task": 27557.489898989897, "last_run_timestamp": "2026-10-03T17:44:09.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8507314900265956, "accuracy_std_dev": 0.014339170843684643, "run_count": 8, "total_questions": 1504, "avg_cost_per_task_usd": 0.018270000664893617, "avg_duration_per_task_ms": 282286.72473404254, "avg_output_tokens_per_task": 34448.984707446805, "last_run_timestamp": "2026-10-03T09:44:00.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.850575, "accuracy_std_dev": null, "run_count": 1, "total_questions": 145, "avg_cost_per_task_usd": 0.015425489655172412, "avg_duration_per_task_ms": 62324.744827586204, "avg_output_tokens_per_task": 15913.096551724138, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8505745613793103, "accuracy_std_dev": 0.00859156025430222, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.05167572781609195, "avg_duration_per_task_ms": 279727.17655172414, "avg_output_tokens_per_task": 46309.96643678161, "last_run_timestamp": "2026-10-03T17:05:07.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8504390873786407, "accuracy_std_dev": 0.008326774223864035, "run_count": 11, "total_questions": 2163, "avg_cost_per_task_usd": 0.07101044845122514, "avg_duration_per_task_ms": 216557.44290337493, "avg_output_tokens_per_task": 22089.64678687009, "last_run_timestamp": "2026-10-03T09:46:11.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.850427, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.014483525641025641, "avg_duration_per_task_ms": 276126.60256410256, "avg_output_tokens_per_task": 12035.96794871795, "last_run_timestamp": "2026-07-18T16:53:38.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8503366000000001, "accuracy_std_dev": 0.010921921719184751, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.028464721717171713, "avg_duration_per_task_ms": 704138.8641414142, "avg_output_tokens_per_task": 32456.946464646466, "last_run_timestamp": "2026-10-03T09:34:29.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8499464590032153, "accuracy_std_dev": 0.022950616654380467, "run_count": 7, "total_questions": 1244, "avg_cost_per_task_usd": 0.010361853697749194, "avg_duration_per_task_ms": 360749.26848874596, "avg_output_tokens_per_task": 20512.316720257233, "last_run_timestamp": "2026-10-03T09:28:36.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8498315000000001, "accuracy_std_dev": 0.03952326621940551, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.17816506616161618, "avg_duration_per_task_ms": 45707.15909090909, "avg_output_tokens_per_task": 3356.5939393939393, "last_run_timestamp": "2026-10-03T09:19:41.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8498314, "accuracy_std_dev": 0.02688129444874262, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.15721394040404038, "avg_duration_per_task_ms": 152201.82676767677, "avg_output_tokens_per_task": 15506.534343434343, "last_run_timestamp": "2026-10-03T09:23:24.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.849375142857143, "accuracy_std_dev": 0.05373653913902533, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.09692596960486322, "avg_duration_per_task_ms": 658494.2613981763, "avg_output_tokens_per_task": 27042.529888551166, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8493265454545452, "accuracy_std_dev": 0.019857803633761332, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.10014188016528924, "avg_duration_per_task_ms": 590927.261707989, "avg_output_tokens_per_task": 43625.35996326905, "last_run_timestamp": "2026-10-03T16:57:15.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.8493151205479452, "accuracy_std_dev": 0.01972059190006447, "run_count": 3, "total_questions": 365, "avg_cost_per_task_usd": 0.06587333698630138, "avg_duration_per_task_ms": 279697.01369863015, "avg_output_tokens_per_task": 16490.610958904108, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8492828305531167, "accuracy_std_dev": 0.030679553487986455, "run_count": 6, "total_questions": 1139, "avg_cost_per_task_usd": 0.1125236496927129, "avg_duration_per_task_ms": 549482.1018437225, "avg_output_tokens_per_task": 33342.05618964003, "last_run_timestamp": "2026-10-03T10:16:13.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8487653333333334, "accuracy_std_dev": 0.011791380929400174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01937495622895623, "avg_duration_per_task_ms": 442506.297979798, "avg_output_tokens_per_task": 38526.9595959596, "last_run_timestamp": "2026-10-03T09:49:43.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8486995349544073, "accuracy_std_dev": 0.009718197845279743, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.10196987639311045, "avg_duration_per_task_ms": 302885.430597771, "avg_output_tokens_per_task": 30304.676798378925, "last_run_timestamp": "2026-10-03T09:33:45.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.848597158186864, "accuracy_std_dev": 0.013754474089263532, "run_count": 6, "total_questions": 1081, "avg_cost_per_task_usd": 0.006980295097132284, "avg_duration_per_task_ms": 490407.8658649399, "avg_output_tokens_per_task": 17200.208140610546, "last_run_timestamp": "2026-10-03T16:56:50.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.26182384343434345, "avg_duration_per_task_ms": 437826.7272727273, "avg_output_tokens_per_task": 59251.080808080806, "last_run_timestamp": "2026-10-06T00:05:36.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8480139539435808, "accuracy_std_dev": 0.013068837655060239, "run_count": 9, "total_questions": 1737, "avg_cost_per_task_usd": 0.02671003742084053, "avg_duration_per_task_ms": 67728.42774899252, "avg_output_tokens_per_task": 19651.77605066206, "last_run_timestamp": "2026-10-03T09:55:20.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8478198075949368, "accuracy_std_dev": 0.015598435238474664, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.07492492320675105, "avg_duration_per_task_ms": 285938.56540084386, "avg_output_tokens_per_task": 19472.943459915612, "last_run_timestamp": "2026-10-03T10:12:41.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8478035581395349, "accuracy_std_dev": 0.061331422730297465, "run_count": 7, "total_questions": 1290, "avg_cost_per_task_usd": 0.010003304651162791, "avg_duration_per_task_ms": 534481.4193798449, "avg_output_tokens_per_task": 19807.151937984498, "last_run_timestamp": "2026-10-03T10:05:20.841Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8477956427969672, "accuracy_std_dev": 0.03319167551259403, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.09577104128053918, "avg_duration_per_task_ms": 1415253.807919124, "avg_output_tokens_per_task": 42907.37742207245, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8477489772382398, "accuracy_std_dev": 0.010277326771101524, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.030415789074355083, "avg_duration_per_task_ms": 680083.3545776429, "avg_output_tokens_per_task": 31761.424380374305, "last_run_timestamp": "2026-10-03T10:03:21.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8476779139314369, "accuracy_std_dev": 0.012065198415953493, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.007001258205689278, "avg_duration_per_task_ms": 526965.3136396791, "avg_output_tokens_per_task": 17331.969365426696, "last_run_timestamp": "2026-10-03T17:01:29.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8474553149480417, "accuracy_std_dev": 0.0229310901226483, "run_count": 13, "total_questions": 2502, "avg_cost_per_task_usd": 0.1667569224620304, "avg_duration_per_task_ms": 293740.35211830534, "avg_output_tokens_per_task": 41652.80575539568, "last_run_timestamp": "2026-10-03T17:08:51.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8467171000000001, "accuracy_std_dev": 0.010298635807231945, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09985031666666667, "avg_duration_per_task_ms": 586132.338888889, "avg_output_tokens_per_task": 31406.80808080808, "last_run_timestamp": "2026-10-03T18:05:42.597Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8466377271407838, "accuracy_std_dev": 0.016590356861005438, "run_count": 7, "total_questions": 1378, "avg_cost_per_task_usd": 0.009304855587808419, "avg_duration_per_task_ms": 444689.5050798258, "avg_output_tokens_per_task": 28194.9412191582, "last_run_timestamp": "2026-10-03T10:04:48.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8466045281660314, "accuracy_std_dev": 0.011422067985981737, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.1267483756882677, "avg_duration_per_task_ms": 355086.75984752225, "avg_output_tokens_per_task": 33407.23930537908, "last_run_timestamp": "2026-10-03T09:40:39.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8464646, "accuracy_std_dev": 0.005966271485609766, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10013606363636364, "avg_duration_per_task_ms": 581159.3505050505, "avg_output_tokens_per_task": 31964.29090909091, "last_run_timestamp": "2026-10-03T17:01:05.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8461756369863013, "accuracy_std_dev": 0.003462455005724478, "run_count": 3, "total_questions": 584, "avg_cost_per_task_usd": 0.07262447602739726, "avg_duration_per_task_ms": 181056.27226027398, "avg_output_tokens_per_task": 31634.909246575342, "last_run_timestamp": "2026-10-03T09:37:27.212Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8461699999999999, "accuracy_std_dev": 0.03781276874945815, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6916325820707071, "avg_duration_per_task_ms": 471363.1818181818, "avg_output_tokens_per_task": 27729.57702020202, "last_run_timestamp": "2026-10-03T09:31:25.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8458659999999999, "accuracy_std_dev": 0.07956358358082058, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007138646464646464, "avg_duration_per_task_ms": 679204.691919192, "avg_output_tokens_per_task": 29375.537598204264, "last_run_timestamp": "2026-10-06T03:03:44.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8454851888607595, "accuracy_std_dev": 0.015145407351405253, "run_count": 10, "total_questions": 1975, "avg_cost_per_task_usd": 0.08874519088607594, "avg_duration_per_task_ms": 411899.82784810127, "avg_output_tokens_per_task": 22073.469873417722, "last_run_timestamp": "2026-10-03T17:44:12.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8453983333333333, "accuracy_std_dev": 0.006102318839770835, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.009480049663299664, "avg_duration_per_task_ms": 418423.1557239057, "avg_output_tokens_per_task": 18452.791245791246, "last_run_timestamp": "2026-10-03T17:22:47.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8453581428571427, "accuracy_std_dev": 0.013659783740481936, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10694367099567102, "avg_duration_per_task_ms": 329051.74314574315, "avg_output_tokens_per_task": 29197.39393939394, "last_run_timestamp": "2026-10-03T09:28:28.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8449273065217391, "accuracy_std_dev": 0.015410014700227412, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.03735347173913043, "avg_duration_per_task_ms": 618214.4876811594, "avg_output_tokens_per_task": 34942.45652173913, "last_run_timestamp": "2026-10-03T17:17:28.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8447187666034155, "accuracy_std_dev": 0.033801436872417136, "run_count": 5, "total_questions": 527, "avg_cost_per_task_usd": 0.10369237191650856, "avg_duration_per_task_ms": 295443.4819734345, "avg_output_tokens_per_task": 23792.745730550283, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8446586363636364, "accuracy_std_dev": 0.08451124453669165, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09526816483011938, "avg_duration_per_task_ms": 784606.3654729109, "avg_output_tokens_per_task": 42179.34618916437, "last_run_timestamp": "2026-10-05T21:07:37.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8444975263157894, "accuracy_std_dev": 0.05215908171451967, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.18362076209463052, "avg_duration_per_task_ms": 40742.116427432215, "avg_output_tokens_per_task": 3305.715842636895, "last_run_timestamp": "2026-10-03T09:56:46.597Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8444072953657173, "accuracy_std_dev": 0.011477740217046216, "run_count": 10, "total_questions": 1791, "avg_cost_per_task_usd": 0.015475027359017308, "avg_duration_per_task_ms": 603617.405360134, "avg_output_tokens_per_task": 16554.757118927973, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8441076000000001, "accuracy_std_dev": 0.00842413538827578, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1428492090909091, "avg_duration_per_task_ms": 160872.74545454545, "avg_output_tokens_per_task": 32292.333333333332, "last_run_timestamp": "2026-10-03T09:56:48.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8441076, "accuracy_std_dev": 0.036707935445077826, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015406010101010103, "avg_duration_per_task_ms": 387191.47878787876, "avg_output_tokens_per_task": 27797.58787878788, "last_run_timestamp": "2026-10-03T16:57:48.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8440029006622516, "accuracy_std_dev": 0.014256857466367755, "run_count": 7, "total_questions": 1359, "avg_cost_per_task_usd": 0.08230250993377485, "avg_duration_per_task_ms": 942574.9713024283, "avg_output_tokens_per_task": 41739.75423105225, "last_run_timestamp": "2026-10-04T00:11:46.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8439016603498544, "accuracy_std_dev": 0.05982632132724711, "run_count": 7, "total_questions": 1372, "avg_cost_per_task_usd": 0.07957356997084548, "avg_duration_per_task_ms": 563629.0634110788, "avg_output_tokens_per_task": 22172.81559766764, "last_run_timestamp": "2026-10-05T21:06:37.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8438616803537586, "accuracy_std_dev": 0.021039203126247442, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.021147979153506004, "avg_duration_per_task_ms": 446490.1522425774, "avg_output_tokens_per_task": 40034.66519267214, "last_run_timestamp": "2026-10-08T06:00:12.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8438314193103448, "accuracy_std_dev": 0.03520063790838207, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.05348304597701149, "avg_duration_per_task_ms": 217549.43770114944, "avg_output_tokens_per_task": 22183.33103448276, "last_run_timestamp": "2026-10-03T10:19:52.322Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8435766681127982, "accuracy_std_dev": 0.022602212098108688, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.2387051561822126, "avg_duration_per_task_ms": 512529.7343938298, "avg_output_tokens_per_task": 45605.27524704748, "last_run_timestamp": "2026-10-09T21:00:06.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8435523956204379, "accuracy_std_dev": 0.013812717451238726, "run_count": 7, "total_questions": 1370, "avg_cost_per_task_usd": 0.13016191605839417, "avg_duration_per_task_ms": 317794.05401459854, "avg_output_tokens_per_task": 27511.544525547444, "last_run_timestamp": "2026-10-04T21:02:19.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8434342499999999, "accuracy_std_dev": 0.008881818827667008, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.14890805050505052, "avg_duration_per_task_ms": 162687.35606060605, "avg_output_tokens_per_task": 32857.1712962963, "last_run_timestamp": "2026-10-03T10:18:11.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8430214877843304, "accuracy_std_dev": 0.03701709147324906, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.16848753580454928, "avg_duration_per_task_ms": 976236.7017691659, "avg_output_tokens_per_task": 53637.603201347934, "last_run_timestamp": "2026-10-03T16:47:29.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8422559000000002, "accuracy_std_dev": 0.012844421749927075, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.13039331515151514, "avg_duration_per_task_ms": 426054.8303030303, "avg_output_tokens_per_task": 36347.83939393939, "last_run_timestamp": "2026-10-03T16:40:33.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8422381263537906, "accuracy_std_dev": 0.013606259020414556, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.1362164772563177, "avg_duration_per_task_ms": 189944.0108303249, "avg_output_tokens_per_task": 30733.362454873648, "last_run_timestamp": "2026-10-03T09:32:18.488Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8419940950570343, "accuracy_std_dev": 0.005052124553096465, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.02034613561470215, "avg_duration_per_task_ms": 621368.3954372624, "avg_output_tokens_per_task": 23198.26235741445, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8419798924812032, "accuracy_std_dev": 0.0161432408803948, "run_count": 7, "total_questions": 1330, "avg_cost_per_task_usd": 0.01795621127819549, "avg_duration_per_task_ms": 242875.14962406014, "avg_output_tokens_per_task": 35697.367669172934, "last_run_timestamp": "2026-10-03T09:24:57.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.841843140655106, "accuracy_std_dev": 0.013637590375502584, "run_count": 6, "total_questions": 1038, "avg_cost_per_task_usd": 0.006975833333333333, "avg_duration_per_task_ms": 638052.9354527938, "avg_output_tokens_per_task": 17230.314065510596, "last_run_timestamp": "2026-10-03T17:20:08.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8417508571428571, "accuracy_std_dev": 0.025909470769412114, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.06952829942279942, "avg_duration_per_task_ms": 590279.7936507936, "avg_output_tokens_per_task": 22093.862914862915, "last_run_timestamp": "2026-10-03T18:21:53.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8417268575037897, "accuracy_std_dev": 0.03274733184905639, "run_count": 30, "total_questions": 5937, "avg_cost_per_task_usd": 0.16544357267980464, "avg_duration_per_task_ms": 149160.101061142, "avg_output_tokens_per_task": 15291.950816910898, "last_run_timestamp": "2026-10-03T10:45:13.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8414701666666667, "accuracy_std_dev": 0.04102371415582561, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.15670842255892256, "avg_duration_per_task_ms": 1400165.1641414142, "avg_output_tokens_per_task": 51380.03956228956, "last_run_timestamp": "2026-10-03T17:37:27.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8414065844994617, "accuracy_std_dev": 0.024904832897923005, "run_count": 7, "total_questions": 929, "avg_cost_per_task_usd": 0.01384601076426265, "avg_duration_per_task_ms": 73903.2508073197, "avg_output_tokens_per_task": 11864.46393972013, "last_run_timestamp": "2026-10-03T17:24:07.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8413977096774194, "accuracy_std_dev": 0.007842951293996439, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.03688066532258065, "avg_duration_per_task_ms": 47685.283602150535, "avg_output_tokens_per_task": 14664.099462365592, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8411896666666667, "accuracy_std_dev": 0.008729468916008341, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.036658867003367, "avg_duration_per_task_ms": 51092.77104377104, "avg_output_tokens_per_task": 14574.808080808081, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8409360293968577, "accuracy_std_dev": 0.016369283081430274, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.2137119655347187, "avg_duration_per_task_ms": 299955.6538266599, "avg_output_tokens_per_task": 48480.95995945261, "last_run_timestamp": "2026-10-03T16:37:10.188Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.840817973713034, "accuracy_std_dev": 0.020791979459397293, "run_count": 5, "total_questions": 913, "avg_cost_per_task_usd": 0.1426375947426068, "avg_duration_per_task_ms": 229737.46549835705, "avg_output_tokens_per_task": 33714.36035049288, "last_run_timestamp": "2026-10-03T09:33:59.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.84048825, "accuracy_std_dev": 0.004667830993887862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06197197222222223, "avg_duration_per_task_ms": 84839.70707070707, "avg_output_tokens_per_task": 13643.420454545454, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8402775531250001, "accuracy_std_dev": 0.007466846475587956, "run_count": 10, "total_questions": 1920, "avg_cost_per_task_usd": 0.039408469791666664, "avg_duration_per_task_ms": 51791.722395833334, "avg_output_tokens_per_task": 14757.431770833333, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8402425498181818, "accuracy_std_dev": 0.00976753775827742, "run_count": 8, "total_questions": 1375, "avg_cost_per_task_usd": 0.0796483170909091, "avg_duration_per_task_ms": 509723.63709090906, "avg_output_tokens_per_task": 19683.845818181817, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8399772366412214, "accuracy_std_dev": 0.010642150299634022, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.02758528159457167, "avg_duration_per_task_ms": 524755.5767599661, "avg_output_tokens_per_task": 22800.932994062765, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8392069911894273, "accuracy_std_dev": 0.01727950565595804, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.03353528046989722, "avg_duration_per_task_ms": 547187.9052863437, "avg_output_tokens_per_task": 32760.090308370043, "last_run_timestamp": "2026-10-03T16:57:14.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8391689163108454, "accuracy_std_dev": 0.01637841911378101, "run_count": 6, "total_questions": 1171, "avg_cost_per_task_usd": 0.014026462852263022, "avg_duration_per_task_ms": 450960.66951323656, "avg_output_tokens_per_task": 38615.76686592656, "last_run_timestamp": "2026-10-03T09:28:28.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8390219978213507, "accuracy_std_dev": 0.019235701159615707, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.047738670297748725, "avg_duration_per_task_ms": 642207.128540305, "avg_output_tokens_per_task": 37915.77995642702, "last_run_timestamp": "2026-10-03T16:58:17.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8389739548751007, "accuracy_std_dev": 0.023840636777552385, "run_count": 7, "total_questions": 1241, "avg_cost_per_task_usd": 0.07291945044319098, "avg_duration_per_task_ms": 368316.54069298954, "avg_output_tokens_per_task": 24012.563255439163, "last_run_timestamp": "2026-10-03T16:56:41.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8389463834196891, "accuracy_std_dev": 0.008774616644047783, "run_count": 4, "total_questions": 772, "avg_cost_per_task_usd": 0.02735646632124352, "avg_duration_per_task_ms": 654835.9339378239, "avg_output_tokens_per_task": 14525.401554404145, "last_run_timestamp": "2026-10-08T23:59:53.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.838739, "accuracy_std_dev": null, "run_count": 1, "total_questions": 185, "avg_cost_per_task_usd": 0.0289266, "avg_duration_per_task_ms": 182176.34594594594, "avg_output_tokens_per_task": 24823.56216216216, "last_run_timestamp": "2026-07-18T08:31:24.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.8385708888888888, "accuracy_std_dev": 0.008600437423558384, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.06734114758698093, "avg_duration_per_task_ms": 345902.8209876543, "avg_output_tokens_per_task": 22364.55892255892, "last_run_timestamp": "2026-10-04T12:03:28.619Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8383837142857142, "accuracy_std_dev": 0.026235324298762266, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11822585137085138, "avg_duration_per_task_ms": 408512.2987012987, "avg_output_tokens_per_task": 35583.771284271286, "last_run_timestamp": "2026-10-03T10:08:41.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8381875026968716, "accuracy_std_dev": 0.003092760668974499, "run_count": 6, "total_questions": 927, "avg_cost_per_task_usd": 0.13592745091693637, "avg_duration_per_task_ms": 121247.60086299892, "avg_output_tokens_per_task": 30661.07551240561, "last_run_timestamp": "2026-10-03T22:03:15.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.837813550776583, "accuracy_std_dev": 0.014764985758205111, "run_count": 10, "total_questions": 1674, "avg_cost_per_task_usd": 0.09487115710872161, "avg_duration_per_task_ms": 634750.7311827957, "avg_output_tokens_per_task": 29572.94982078853, "last_run_timestamp": "2026-10-08T23:03:31.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.837571, "accuracy_std_dev": null, "run_count": 1, "total_questions": 118, "avg_cost_per_task_usd": 0.01632071186440678, "avg_duration_per_task_ms": 269506.5254237288, "avg_output_tokens_per_task": 11788.906779661016, "last_run_timestamp": "2026-07-18T16:47:35.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8375589327830189, "accuracy_std_dev": 0.020056898198209345, "run_count": 9, "total_questions": 1696, "avg_cost_per_task_usd": 0.054286748820754724, "avg_duration_per_task_ms": 282159.9534198113, "avg_output_tokens_per_task": 22660.685141509435, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8375548434650456, "accuracy_std_dev": 0.05830789076754535, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.0750043009118541, "avg_duration_per_task_ms": 671651.9898682877, "avg_output_tokens_per_task": 45629.63880445795, "last_run_timestamp": "2026-10-03T17:06:14.822Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8374960027752081, "accuracy_std_dev": 0.029394078011376546, "run_count": 6, "total_questions": 1081, "avg_cost_per_task_usd": 0.0877097058279371, "avg_duration_per_task_ms": 1673535.2590194265, "avg_output_tokens_per_task": 40889.8316373728, "last_run_timestamp": "2026-08-29T19:12:33.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8371595454545454, "accuracy_std_dev": 0.01566308613131605, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.07092007805325987, "avg_duration_per_task_ms": 680254.4848484849, "avg_output_tokens_per_task": 53261.87281910009, "last_run_timestamp": "2026-10-03T16:57:33.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.8367169999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.10272846700507614, "avg_duration_per_task_ms": 406781.3959390863, "avg_output_tokens_per_task": 31764.908629441623, "last_run_timestamp": "2026-07-18T08:30:34.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8367001428571429, "accuracy_std_dev": 0.02855562409668226, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14463846464646468, "avg_duration_per_task_ms": 904627.3023088023, "avg_output_tokens_per_task": 54000.30014430014, "last_run_timestamp": "2026-10-03T17:37:18.132Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8366436799091941, "accuracy_std_dev": 0.015820054069611535, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.08522557775255392, "avg_duration_per_task_ms": 546245.1770715096, "avg_output_tokens_per_task": 31173.529511918274, "last_run_timestamp": "2026-10-03T11:03:00.010Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8361980881913303, "accuracy_std_dev": 0.014593793273938368, "run_count": 7, "total_questions": 1338, "avg_cost_per_task_usd": 0.21788730717488788, "avg_duration_per_task_ms": 418506.8639760837, "avg_output_tokens_per_task": 49387.9245142003, "last_run_timestamp": "2026-10-03T16:59:26.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8361952000000001, "accuracy_std_dev": 0.0746685963545586, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.19211891767676767, "avg_duration_per_task_ms": 535463.2151515152, "avg_output_tokens_per_task": 43715.205555555556, "last_run_timestamp": "2026-10-03T18:53:59.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8360065448008849, "accuracy_std_dev": 0.008779454087962683, "run_count": 11, "total_questions": 1808, "avg_cost_per_task_usd": 0.018033527101769913, "avg_duration_per_task_ms": 187582.5735619469, "avg_output_tokens_per_task": 35877.24225663717, "last_run_timestamp": "2026-10-03T09:31:41.635Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.8357181666666667, "accuracy_std_dev": 0.013680371820198815, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006888778619528619, "avg_duration_per_task_ms": 375845.8905723906, "avg_output_tokens_per_task": 26123.255892255893, "last_run_timestamp": "2026-10-03T17:23:05.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.835648125, "accuracy_std_dev": 0.012901937649414363, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03973862878787879, "avg_duration_per_task_ms": 223092.5542929293, "avg_output_tokens_per_task": 14589.925505050505, "last_run_timestamp": "2026-10-03T09:51:44.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8354408285280729, "accuracy_std_dev": 0.043474653742611905, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.1658476181082448, "avg_duration_per_task_ms": 161596.665149216, "avg_output_tokens_per_task": 16364.909458775923, "last_run_timestamp": "2026-10-03T09:33:36.803Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8353909418724279, "accuracy_std_dev": 0.02098218819975649, "run_count": 10, "total_questions": 1944, "avg_cost_per_task_usd": 0.21285450617283952, "avg_duration_per_task_ms": 291168.728909465, "avg_output_tokens_per_task": 48204.40740740741, "last_run_timestamp": "2026-10-03T16:57:30.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8350166666666667, "accuracy_std_dev": 0.019584493003961645, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.025601432659932657, "avg_duration_per_task_ms": 248785.20707070708, "avg_output_tokens_per_task": 21309.069865319867, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8350166666666667, "accuracy_std_dev": 0.005991526757754575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06194655555555556, "avg_duration_per_task_ms": 84396.87373737374, "avg_output_tokens_per_task": 13638.5, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8348964285714285, "accuracy_std_dev": 0.013870860266422272, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13617697113997115, "avg_duration_per_task_ms": 172785.43217893218, "avg_output_tokens_per_task": 30709.33621933622, "last_run_timestamp": "2026-10-03T09:19:24.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.834704304347826, "accuracy_std_dev": 0.01114896903933274, "run_count": 5, "total_questions": 851, "avg_cost_per_task_usd": 0.04789096122209165, "avg_duration_per_task_ms": 91498.8401880141, "avg_output_tokens_per_task": 15123.290246768507, "last_run_timestamp": "2026-10-09T02:08:01.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8345736842105264, "accuracy_std_dev": 0.012051692648327507, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.2574891374269006, "avg_duration_per_task_ms": 275645.54146730463, "avg_output_tokens_per_task": 56198.80940988836, "last_run_timestamp": "2026-10-03T17:33:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8344556666666666, "accuracy_std_dev": 0.04246379448685927, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1074556616161616, "avg_duration_per_task_ms": 1223271.058922559, "avg_output_tokens_per_task": 43332.00420875421, "last_run_timestamp": "2026-08-29T18:42:20.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "WandB", "accuracy": 0.834197, "accuracy_std_dev": null, "run_count": 1, "total_questions": 193, "avg_cost_per_task_usd": 0.02599202590673575, "avg_duration_per_task_ms": 128468.99481865285, "avg_output_tokens_per_task": 20660.492227979274, "last_run_timestamp": "2026-07-18T16:00:08.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8341842341356673, "accuracy_std_dev": 0.02342272596676753, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.08859786870897156, "avg_duration_per_task_ms": 985152.2742523706, "avg_output_tokens_per_task": 40722.371991247266, "last_run_timestamp": "2026-10-04T00:10:49.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8341792639593909, "accuracy_std_dev": 0.020282949394476377, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.07842123350253806, "avg_duration_per_task_ms": 728101.9196277496, "avg_output_tokens_per_task": 42337.8130287648, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8340809047085203, "accuracy_std_dev": 0.01732897775403963, "run_count": 5, "total_questions": 892, "avg_cost_per_task_usd": 0.020895510089686098, "avg_duration_per_task_ms": 283776.2948430493, "avg_output_tokens_per_task": 41571.40470852018, "last_run_timestamp": "2026-10-03T09:31:15.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8340712409867174, "accuracy_std_dev": 0.026087244621414103, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.15789761669829225, "avg_duration_per_task_ms": 841435.927261227, "avg_output_tokens_per_task": 51847.42757748261, "last_run_timestamp": "2026-10-03T17:11:42.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.05954210180106922, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0609229090909091, "avg_duration_per_task_ms": 629153.6734006734, "avg_output_tokens_per_task": 22578.223905723906, "last_run_timestamp": "2026-10-03T16:58:19.180Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8337931059602649, "accuracy_std_dev": 0.013355036895202588, "run_count": 12, "total_questions": 1812, "avg_cost_per_task_usd": 0.03549873675496689, "avg_duration_per_task_ms": 241451.8725165563, "avg_output_tokens_per_task": 28296.954194260485, "last_run_timestamp": "2026-10-03T16:12:07.493Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8334570734966593, "accuracy_std_dev": 0.0419490749322732, "run_count": 7, "total_questions": 1347, "avg_cost_per_task_usd": 0.02924405196733482, "avg_duration_per_task_ms": 281996.3355605048, "avg_output_tokens_per_task": 9032.244988864142, "last_run_timestamp": "2026-10-03T10:36:42.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.8333335829015545, "accuracy_std_dev": 0.019006499999999982, "run_count": 2, "total_questions": 386, "avg_cost_per_task_usd": 0.023828624352331605, "avg_duration_per_task_ms": 290054.69430051814, "avg_output_tokens_per_task": 19802.816062176167, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8329019637618638, "accuracy_std_dev": 0.014650670481433776, "run_count": 6, "total_questions": 1159, "avg_cost_per_task_usd": 0.1486290198446937, "avg_duration_per_task_ms": 266639.7955133736, "avg_output_tokens_per_task": 33590.909404659185, "last_run_timestamp": "2026-10-03T10:00:12.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.832620296969697, "accuracy_std_dev": 0.019550200858985223, "run_count": 15, "total_questions": 2805, "avg_cost_per_task_usd": 0.1869388680926916, "avg_duration_per_task_ms": 373798.6777183601, "avg_output_tokens_per_task": 38438.90623885918, "last_run_timestamp": "2026-10-03T16:59:44.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.8325081287128712, "accuracy_std_dev": 0.06854471399887452, "run_count": 4, "total_questions": 606, "avg_cost_per_task_usd": 0.13443122607260727, "avg_duration_per_task_ms": 1513362.3762376239, "avg_output_tokens_per_task": 46533.88613861386, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8324915000000002, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13435382575757576, "avg_duration_per_task_ms": 466692.7878787879, "avg_output_tokens_per_task": 61036.83838383838, "last_run_timestamp": "2026-10-03T17:03:50.322Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8324914999999999, "accuracy_std_dev": 0.01902788623310536, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02279124747474747, "avg_duration_per_task_ms": 532516.4444444445, "avg_output_tokens_per_task": 41107.4779040404, "last_run_timestamp": "2026-10-03T10:07:54.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.8324914999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023044303030303033, "avg_duration_per_task_ms": 640134.2070707071, "avg_output_tokens_per_task": 54011.681818181816, "last_run_timestamp": "2026-09-26T08:54:28.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8323712857142858, "accuracy_std_dev": 0.006903653726506745, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.24708544516594516, "avg_duration_per_task_ms": 346092.02308802307, "avg_output_tokens_per_task": 40400.90404040404, "last_run_timestamp": "2026-10-03T10:13:08.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8323231, "accuracy_std_dev": 0.008045717599443841, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.08128380757575758, "avg_duration_per_task_ms": 799166.1005050505, "avg_output_tokens_per_task": 48099.10101010101, "last_run_timestamp": "2026-10-03T16:25:22.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8321069434069044, "accuracy_std_dev": 0.013054586209736332, "run_count": 11, "total_questions": 1767, "avg_cost_per_task_usd": 0.11444472948500282, "avg_duration_per_task_ms": 230736.259762309, "avg_output_tokens_per_task": 27650.263157894737, "last_run_timestamp": "2026-10-09T15:49:59.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8314979893324156, "accuracy_std_dev": 0.10730469177701411, "run_count": 15, "total_questions": 2906, "avg_cost_per_task_usd": 0.007892868203716447, "avg_duration_per_task_ms": 643389.9057123193, "avg_output_tokens_per_task": 27155.329318651067, "last_run_timestamp": "2026-10-03T09:52:45.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8310885000000001, "accuracy_std_dev": 0.012955632388399007, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06418621717171719, "avg_duration_per_task_ms": 117116.24831649831, "avg_output_tokens_per_task": 13483.090067340067, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8309870236486486, "accuracy_std_dev": 0.03224080320546207, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.015772429054054054, "avg_duration_per_task_ms": 136867.50112612612, "avg_output_tokens_per_task": 17821.467905405407, "last_run_timestamp": "2026-10-05T18:01:16.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8308079999999999, "accuracy_std_dev": 0.019360000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.16246437626262625, "avg_duration_per_task_ms": 214673.86111111112, "avg_output_tokens_per_task": 36669.38383838384, "last_run_timestamp": "2026-10-03T09:42:09.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8307627712082263, "accuracy_std_dev": 0.005791790283452965, "run_count": 8, "total_questions": 1556, "avg_cost_per_task_usd": 0.03552349742930591, "avg_duration_per_task_ms": 197284.01863753214, "avg_output_tokens_per_task": 13956.56233933162, "last_run_timestamp": "2026-10-03T09:16:27.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.8303493580562661, "accuracy_std_dev": 0.021497934689561845, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.0706463444160273, "avg_duration_per_task_ms": 871151.2676896845, "avg_output_tokens_per_task": 40306.14833759591, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8302722727272728, "accuracy_std_dev": 0.03337639966446328, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.05463231588613406, "avg_duration_per_task_ms": 268557.7502295684, "avg_output_tokens_per_task": 23385.22819100092, "last_run_timestamp": "2026-10-03T09:29:36.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.829966, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022576904040404043, "avg_duration_per_task_ms": 10045.939393939394, "avg_output_tokens_per_task": 14604.111111111111, "last_run_timestamp": "2026-10-03T08:20:05.754Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.8298668364249578, "accuracy_std_dev": 0.03414970870688992, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.10817975548060708, "avg_duration_per_task_ms": 980232.0573355817, "avg_output_tokens_per_task": 33808.60314783586, "last_run_timestamp": "2026-09-26T10:24:21.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8296294999999999, "accuracy_std_dev": 0.023614739190810495, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.22653554494949493, "avg_duration_per_task_ms": 333936.53939393937, "avg_output_tokens_per_task": 51370.22727272727, "last_run_timestamp": "2026-10-03T17:09:33.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8292847495662232, "accuracy_std_dev": 0.024481615398972544, "run_count": 9, "total_questions": 1729, "avg_cost_per_task_usd": 0.021951503759398496, "avg_duration_per_task_ms": 881311.056680162, "avg_output_tokens_per_task": 45066.87854251012, "last_run_timestamp": "2026-10-03T09:15:02.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8289733974683544, "accuracy_std_dev": 0.06161366912305707, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.22182355358649788, "avg_duration_per_task_ms": 1441908.6481012658, "avg_output_tokens_per_task": 52295.71561181435, "last_run_timestamp": "2026-10-03T17:35:18.039Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8288054812419146, "accuracy_std_dev": 0.014299354744511224, "run_count": 8, "total_questions": 1546, "avg_cost_per_task_usd": 0.03668785575679172, "avg_duration_per_task_ms": 429133.35187580856, "avg_output_tokens_per_task": 46204.894566623545, "last_run_timestamp": "2026-10-03T09:52:10.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8283596285714285, "accuracy_std_dev": 0.008957848077942624, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.007575441269841269, "avg_duration_per_task_ms": 331346.21904761903, "avg_output_tokens_per_task": 19200.022857142856, "last_run_timestamp": "2026-10-04T00:00:36.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.828283, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009031212121212122, "avg_duration_per_task_ms": 399484.5707070707, "avg_output_tokens_per_task": 21544.277777777777, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8282826999999999, "accuracy_std_dev": 0.026967609579085804, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.13044083383838387, "avg_duration_per_task_ms": 536030.553030303, "avg_output_tokens_per_task": 39996.7904040404, "last_run_timestamp": "2026-10-03T16:58:32.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.827862, "accuracy_std_dev": 0.0497186507736282, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.16292082386363638, "avg_duration_per_task_ms": 160402.9981060606, "avg_output_tokens_per_task": 16138.333333333334, "last_run_timestamp": "2026-10-03T09:35:14.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8275666632911391, "accuracy_std_dev": 0.031862730949862604, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.025537988185654007, "avg_duration_per_task_ms": 492003.4506329114, "avg_output_tokens_per_task": 23813.98734177215, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.82744125, "accuracy_std_dev": 0.11256498383905848, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.10842245328282829, "avg_duration_per_task_ms": 1501711.7184343433, "avg_output_tokens_per_task": 26982.195707070707, "last_run_timestamp": "2026-10-03T16:52:29.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8274407499999998, "accuracy_std_dev": 0.002791768917281665, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06294326136363637, "avg_duration_per_task_ms": 238417.01893939395, "avg_output_tokens_per_task": 15108.617424242424, "last_run_timestamp": "2026-10-03T10:04:25.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8274200658895084, "accuracy_std_dev": 0.03473866469065846, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.06533462037506337, "avg_duration_per_task_ms": 1047959.904713634, "avg_output_tokens_per_task": 32930.106943740495, "last_run_timestamp": "2026-10-03T17:39:05.807Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.826839857142857, "accuracy_std_dev": 0.052812585974039414, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09028968326118327, "avg_duration_per_task_ms": 468556.38167388167, "avg_output_tokens_per_task": 20487.358585858587, "last_run_timestamp": "2026-10-03T17:40:21.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8267676, "accuracy_std_dev": 0.014373097433747551, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.06195891818181818, "avg_duration_per_task_ms": 501765.85909090907, "avg_output_tokens_per_task": 48813.566666666666, "last_run_timestamp": "2026-10-03T17:23:58.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8265992857142858, "accuracy_std_dev": 0.05408198806498543, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09954717027417027, "avg_duration_per_task_ms": 788800.0598845598, "avg_output_tokens_per_task": 39778.419913419915, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8265991666666667, "accuracy_std_dev": 0.013432985736817995, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024518459595959594, "avg_duration_per_task_ms": 477551.41245791246, "avg_output_tokens_per_task": 22795.994107744107, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.8265989999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02378419696969697, "avg_duration_per_task_ms": 421976.08585858584, "avg_output_tokens_per_task": 20387.959595959597, "last_run_timestamp": "2026-07-18T16:18:27.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8263587142857144, "accuracy_std_dev": 0.007278841366086593, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1363799206349206, "avg_duration_per_task_ms": 152940.04473304472, "avg_output_tokens_per_task": 33517.63347763348, "last_run_timestamp": "2026-10-03T09:28:23.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8262626, "accuracy_std_dev": 0.023626710774883566, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01075091717171717, "avg_duration_per_task_ms": 644954.0035353536, "avg_output_tokens_per_task": 38156.42727272727, "last_run_timestamp": "2026-10-03T09:23:44.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8261182857142857, "accuracy_std_dev": 0.010787762507506711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.043446235209235205, "avg_duration_per_task_ms": 226902.49855699856, "avg_output_tokens_per_task": 14365.386002886004, "last_run_timestamp": "2026-10-03T09:30:12.452Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8259840411061286, "accuracy_std_dev": 0.022038029406052623, "run_count": 14, "total_questions": 2676, "avg_cost_per_task_usd": 0.04315680829596412, "avg_duration_per_task_ms": 371792.15134529147, "avg_output_tokens_per_task": 46418.25635276532, "last_run_timestamp": "2026-10-09T16:15:58.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8258873707224335, "accuracy_std_dev": 0.009676826643527784, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.025350776615969583, "avg_duration_per_task_ms": 120122.08840304182, "avg_output_tokens_per_task": 20151.38783269962, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8254038475770924, "accuracy_std_dev": 0.017871570248271126, "run_count": 6, "total_questions": 1135, "avg_cost_per_task_usd": 0.024938470484581497, "avg_duration_per_task_ms": 321173.1885462555, "avg_output_tokens_per_task": 49662.51806167401, "last_run_timestamp": "2026-10-03T09:30:56.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8253399469387755, "accuracy_std_dev": 0.06202361079266511, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.048117852040816325, "avg_duration_per_task_ms": 1077180.0714285714, "avg_output_tokens_per_task": 26155.976530612246, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.824916, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022821656565656566, "avg_duration_per_task_ms": 516406.73232323234, "avg_output_tokens_per_task": 45403.40909090909, "last_run_timestamp": "2026-10-04T03:04:38.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8249157142857142, "accuracy_std_dev": 0.019837994431424387, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1225871738816739, "avg_duration_per_task_ms": 339042.2626262626, "avg_output_tokens_per_task": 50612.39971139971, "last_run_timestamp": "2026-09-26T08:56:23.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.824777114967462, "accuracy_std_dev": 0.009549509751280208, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.19052757122198116, "avg_duration_per_task_ms": 927003.5198843095, "avg_output_tokens_per_task": 50705.37816341287, "last_run_timestamp": "2026-10-03T16:39:56.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8243546666666666, "accuracy_std_dev": 0.012396478769751095, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.038657538720538724, "avg_duration_per_task_ms": 268369.8787878788, "avg_output_tokens_per_task": 15453.594276094276, "last_run_timestamp": "2026-10-03T09:38:33.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.8243470488301118, "accuracy_std_dev": 0.01897362841419636, "run_count": 5, "total_questions": 983, "avg_cost_per_task_usd": 0.09452765412004069, "avg_duration_per_task_ms": 201477.20142421158, "avg_output_tokens_per_task": 27045.996948118005, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8238636249999999, "accuracy_std_dev": 0.022648157226016746, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03663502967171717, "avg_duration_per_task_ms": 270780.9375, "avg_output_tokens_per_task": 14336.900252525253, "last_run_timestamp": "2026-10-03T09:40:25.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.823569, "accuracy_std_dev": 0.003264448253533828, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.039843759595959594, "avg_duration_per_task_ms": 211450.69292929294, "avg_output_tokens_per_task": 14991.045454545454, "last_run_timestamp": "2026-10-03T09:53:48.526Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8234848000000001, "accuracy_std_dev": 0.06057572578979139, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.05154826161616162, "avg_duration_per_task_ms": 588991.1358585858, "avg_output_tokens_per_task": 24679.851515151517, "last_run_timestamp": "2026-10-03T08:20:09.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8233863549206349, "accuracy_std_dev": 0.011000693398599945, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.025066622222222223, "avg_duration_per_task_ms": 470143.7276190476, "avg_output_tokens_per_task": 25031.77904761905, "last_run_timestamp": "2026-10-08T23:31:01.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.8233255173210161, "accuracy_std_dev": 0.0467612856991764, "run_count": 5, "total_questions": 866, "avg_cost_per_task_usd": 0.16841330484988454, "avg_duration_per_task_ms": 1763547.682448037, "avg_output_tokens_per_task": 49858.13856812933, "last_run_timestamp": "2026-10-03T17:19:48.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8232324285714286, "accuracy_std_dev": 0.007793090691799505, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.048816260461760456, "avg_duration_per_task_ms": 449944.26046176045, "avg_output_tokens_per_task": 27045.45238095238, "last_run_timestamp": "2026-10-03T16:27:31.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025912994949494946, "avg_duration_per_task_ms": 162312.898989899, "avg_output_tokens_per_task": 9036.257575757576, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8231693085966202, "accuracy_std_dev": 0.01720174744208347, "run_count": 7, "total_questions": 1361, "avg_cost_per_task_usd": 0.13977533284349744, "avg_duration_per_task_ms": 264582.22189566493, "avg_output_tokens_per_task": 31542.92799412197, "last_run_timestamp": "2026-10-03T09:52:37.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.823129, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.032144397959183675, "avg_duration_per_task_ms": 345053.9591836735, "avg_output_tokens_per_task": 27884.80612244898, "last_run_timestamp": "2026-07-18T08:29:12.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8231118571428572, "accuracy_std_dev": 0.010745952705601317, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09818569913419915, "avg_duration_per_task_ms": 370933.02958152955, "avg_output_tokens_per_task": 45322.332611832615, "last_run_timestamp": "2026-10-04T00:05:42.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8227512857142858, "accuracy_std_dev": 0.013638696342332996, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03510758802308803, "avg_duration_per_task_ms": 250326.75108225108, "avg_output_tokens_per_task": 13920.564213564214, "last_run_timestamp": "2026-10-03T09:31:25.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8223905999999999, "accuracy_std_dev": 0.023475537008554266, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.03811036666666667, "avg_duration_per_task_ms": 320984.0297979798, "avg_output_tokens_per_task": 39454.10808080808, "last_run_timestamp": "2026-10-03T16:50:17.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007053464646464647, "avg_duration_per_task_ms": 622803.648989899, "avg_output_tokens_per_task": 16887.560606060608, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8223905, "accuracy_std_dev": 0.021663157431839608, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10283124074074075, "avg_duration_per_task_ms": 597427.9688552188, "avg_output_tokens_per_task": 38154.76262626263, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8215404398976982, "accuracy_std_dev": 0.032891408388902345, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.02587831713554987, "avg_duration_per_task_ms": 460054.41943734017, "avg_output_tokens_per_task": 21496.38874680307, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8211085620915033, "accuracy_std_dev": 0.010055056516550273, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.022989802469135803, "avg_duration_per_task_ms": 339486.9368191721, "avg_output_tokens_per_task": 20491.059549745823, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8210436999999998, "accuracy_std_dev": 0.014005426348740687, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.3147447828282829, "avg_duration_per_task_ms": 289184.82020202023, "avg_output_tokens_per_task": 20793.45303030303, "last_run_timestamp": "2026-10-03T09:11:51.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.8208440232558138, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 387, "avg_cost_per_task_usd": 0.007978992248062014, "avg_duration_per_task_ms": 646108.8320413437, "avg_output_tokens_per_task": 19074.485788113696, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.820801884564352, "accuracy_std_dev": 0.01789318941878136, "run_count": 12, "total_questions": 2261, "avg_cost_per_task_usd": 0.03227863688633348, "avg_duration_per_task_ms": 262227.01946041576, "avg_output_tokens_per_task": 14197.505528527201, "last_run_timestamp": "2026-10-09T15:42:38.051Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.82070725, "accuracy_std_dev": 0.0036691031857253717, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1439596148989899, "avg_duration_per_task_ms": 393442.0984848485, "avg_output_tokens_per_task": 28794.420454545456, "last_run_timestamp": "2026-10-03T09:54:41.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8205311347826086, "accuracy_std_dev": 0.003366096503885395, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.007773560869565217, "avg_duration_per_task_ms": 433269.1550724638, "avg_output_tokens_per_task": 19994.68913043478, "last_run_timestamp": "2026-10-04T00:12:29.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8202530101265821, "accuracy_std_dev": 0.008020329102765456, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.018872242194092826, "avg_duration_per_task_ms": 516686.08607594937, "avg_output_tokens_per_task": 18743.98734177215, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.820012649122807, "accuracy_std_dev": 0.022660477062222247, "run_count": 4, "total_questions": 513, "avg_cost_per_task_usd": 0.08905661598440547, "avg_duration_per_task_ms": 310577.5692007797, "avg_output_tokens_per_task": 36897.34307992203, "last_run_timestamp": "2026-10-03T17:40:40.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.819865, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008803729797979798, "avg_duration_per_task_ms": 1022564.7878787878, "avg_output_tokens_per_task": 21115.48232323232, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8193234782608696, "accuracy_std_dev": 0.007347505135369806, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.007821787681159422, "avg_duration_per_task_ms": 636163.7028985508, "avg_output_tokens_per_task": 18195.539130434783, "last_run_timestamp": "2026-10-04T01:46:36.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8193039999999999, "accuracy_std_dev": 0.01926018039029404, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0067461641414141415, "avg_duration_per_task_ms": 634323.356902357, "avg_output_tokens_per_task": 18174.79292929293, "last_run_timestamp": "2026-10-03T17:09:07.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8192329035532995, "accuracy_std_dev": 0.01830389154129678, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.022038197123519458, "avg_duration_per_task_ms": 499412.81133671745, "avg_output_tokens_per_task": 22395.89847715736, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8191919000000001, "accuracy_std_dev": 0.0203168642878275, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.012244186363636363, "avg_duration_per_task_ms": 659276.7813131313, "avg_output_tokens_per_task": 36210.856060606064, "last_run_timestamp": "2026-10-03T10:16:01.896Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8179946666666665, "accuracy_std_dev": 0.06050420910059803, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.027902544332211, "avg_duration_per_task_ms": 645703.9062850729, "avg_output_tokens_per_task": 28951.189674523008, "last_run_timestamp": "2026-10-03T10:05:26.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.8179010000000001, "accuracy_std_dev": 0.032213432208733475, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09231441077441079, "avg_duration_per_task_ms": 1504600.9158249157, "avg_output_tokens_per_task": 43006.99410774411, "last_run_timestamp": "2026-10-03T17:44:28.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.817901, "accuracy_std_dev": null, "run_count": 1, "total_questions": 54, "avg_cost_per_task_usd": 0.008988981481481481, "avg_duration_per_task_ms": 114091.12962962964, "avg_output_tokens_per_task": 16251.87037037037, "last_run_timestamp": "2026-07-18T09:01:08.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8178076666666667, "accuracy_std_dev": 0.020010550661532987, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04244091638608306, "avg_duration_per_task_ms": 255376.0824915825, "avg_output_tokens_per_task": 14176.158810325476, "last_run_timestamp": "2026-10-03T10:33:26.506Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8169791428571429, "accuracy_std_dev": 0.022383364227087234, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04123865007215007, "avg_duration_per_task_ms": 175510.44805194804, "avg_output_tokens_per_task": 22808.474025974025, "last_run_timestamp": "2026-10-03T17:30:27.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8168942867981791, "accuracy_std_dev": 0.04478955350480735, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.08493498533131005, "avg_duration_per_task_ms": 815555.1937278705, "avg_output_tokens_per_task": 31553.02023267577, "last_run_timestamp": "2026-10-03T19:00:29.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8165714782608695, "accuracy_std_dev": 0.012063277850998734, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.3338848906455862, "avg_duration_per_task_ms": 299201.53820816864, "avg_output_tokens_per_task": 21149.658322353975, "last_run_timestamp": "2026-10-03T09:43:11.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8162174999999999, "accuracy_std_dev": 0.009642570572725913, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.019066546296296293, "avg_duration_per_task_ms": 525999.5555555555, "avg_output_tokens_per_task": 18868.49074074074, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.815937, "accuracy_std_dev": 0.021042110493009022, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04137441245791246, "avg_duration_per_task_ms": 82874.20875420875, "avg_output_tokens_per_task": 27465.196969696968, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "WandB", "accuracy": 0.815838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 181, "avg_cost_per_task_usd": 0.03376955248618784, "avg_duration_per_task_ms": 178211.5359116022, "avg_output_tokens_per_task": 26893.3591160221, "last_run_timestamp": "2026-07-18T08:04:57.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8156999235181644, "accuracy_std_dev": 0.04951986238986445, "run_count": 9, "total_questions": 1569, "avg_cost_per_task_usd": 0.04647088910133843, "avg_duration_per_task_ms": 572516.0726577438, "avg_output_tokens_per_task": 26962.156787762906, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8156565, "accuracy_std_dev": 0.01038452301023016, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.36446038207070713, "avg_duration_per_task_ms": 359197.3244949495, "avg_output_tokens_per_task": 22956.0851010101, "last_run_timestamp": "2026-10-03T09:59:43.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8154587608695651, "accuracy_std_dev": 0.035429792247399325, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.07787585652173913, "avg_duration_per_task_ms": 892615.7195652174, "avg_output_tokens_per_task": 39764.602173913045, "last_run_timestamp": "2026-10-04T00:10:24.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8152592970976255, "accuracy_std_dev": 0.06424712746051377, "run_count": 20, "total_questions": 3790, "avg_cost_per_task_usd": 0.007318012664907651, "avg_duration_per_task_ms": 480925.70474934037, "avg_output_tokens_per_task": 17239.7055408971, "last_run_timestamp": "2026-10-08T22:38:34.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.814024, "accuracy_std_dev": null, "run_count": 1, "total_questions": 164, "avg_cost_per_task_usd": 0.010100225609756097, "avg_duration_per_task_ms": 515065.23780487804, "avg_output_tokens_per_task": 33245.84146341463, "last_run_timestamp": "2026-09-28T09:46:11.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8139730000000001, "accuracy_std_dev": 0.007059363734312956, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006141271885521885, "avg_duration_per_task_ms": 521938.39983164985, "avg_output_tokens_per_task": 18689.60521885522, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.813518124726477, "accuracy_std_dev": 0.03129658994855769, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.2371703887673231, "avg_duration_per_task_ms": 955825.8431801605, "avg_output_tokens_per_task": 54088.58570386579, "last_run_timestamp": "2026-10-03T17:05:40.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8133717142857142, "accuracy_std_dev": 0.009499021449064938, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.029931976190476193, "avg_duration_per_task_ms": 201570.87085137086, "avg_output_tokens_per_task": 23744.84992784993, "last_run_timestamp": "2026-10-03T17:45:01.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8131313333333333, "accuracy_std_dev": 0.02212166805545088, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2550713585858586, "avg_duration_per_task_ms": 227479.37878787878, "avg_output_tokens_per_task": 25491.23232323232, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8128504999999999, "accuracy_std_dev": 0.008607050515904558, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03321095707070707, "avg_duration_per_task_ms": 706759.8703703703, "avg_output_tokens_per_task": 15781.767676767677, "last_run_timestamp": "2026-10-03T09:29:30.693Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8122894999999999, "accuracy_std_dev": 0.020078789204448232, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006274926767676768, "avg_duration_per_task_ms": 514195.62205387204, "avg_output_tokens_per_task": 18652.397306397306, "last_run_timestamp": "2026-10-03T16:58:24.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8118160109409189, "accuracy_std_dev": 0.026061913159839602, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.2012101502552881, "avg_duration_per_task_ms": 871685.5309992706, "avg_output_tokens_per_task": 54661.03574033552, "last_run_timestamp": "2026-10-03T17:03:42.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.811448, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004932363636363637, "avg_duration_per_task_ms": 1604879.5959595959, "avg_output_tokens_per_task": 24145.00505050505, "last_run_timestamp": "2026-07-18T08:42:46.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8112072857142857, "accuracy_std_dev": 0.030080678790376323, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014970598124098124, "avg_duration_per_task_ms": 908622.7445887446, "avg_output_tokens_per_task": 44238.631313131315, "last_run_timestamp": "2026-10-03T08:04:44.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8108866666666668, "accuracy_std_dev": 0.013699776466140677, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006567723063973063, "avg_duration_per_task_ms": 517378.0252525252, "avg_output_tokens_per_task": 17537.287037037036, "last_run_timestamp": "2026-10-03T17:03:40.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8103253333333333, "accuracy_std_dev": 0.011527714209195522, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04107354882154882, "avg_duration_per_task_ms": 75169.15151515152, "avg_output_tokens_per_task": 27258.996632996634, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8099964482758623, "accuracy_std_dev": 0.03423948248791152, "run_count": 5, "total_questions": 957, "avg_cost_per_task_usd": 0.07441870846394985, "avg_duration_per_task_ms": 1390345.897596656, "avg_output_tokens_per_task": 42629.52142110763, "last_run_timestamp": "2026-08-22T17:50:04.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.8095960000000001, "accuracy_std_dev": 0.036971651099186785, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.11996981868686869, "avg_duration_per_task_ms": 468196.27575757576, "avg_output_tokens_per_task": 33838.74393939394, "last_run_timestamp": "2026-10-03T16:58:37.335Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8091166153846153, "accuracy_std_dev": 0.03183613126622014, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.19830280854700855, "avg_duration_per_task_ms": 1060541.5008547008, "avg_output_tokens_per_task": 55592.3264957265, "last_run_timestamp": "2026-10-03T16:58:49.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8087628910675383, "accuracy_std_dev": 0.05175650112216013, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.23057392955700798, "avg_duration_per_task_ms": 1023108.186637618, "avg_output_tokens_per_task": 52917.13652868555, "last_run_timestamp": "2026-10-03T17:12:36.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8087552731239093, "accuracy_std_dev": 0.0546199990705989, "run_count": 6, "total_questions": 1146, "avg_cost_per_task_usd": 0.024335609075043632, "avg_duration_per_task_ms": 1150538.0907504363, "avg_output_tokens_per_task": 49299.98254799302, "last_run_timestamp": "2026-10-03T10:11:21.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.808728230769231, "accuracy_std_dev": 0.016181671721919724, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.34003047086247085, "avg_duration_per_task_ms": 354646.60295260296, "avg_output_tokens_per_task": 22500.726495726496, "last_run_timestamp": "2026-10-03T09:08:27.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8085015000000001, "accuracy_std_dev": 0.026787617671230127, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1100901268939394, "avg_duration_per_task_ms": 771715.6508838384, "avg_output_tokens_per_task": 57264.88383838384, "last_run_timestamp": "2026-10-03T17:13:30.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.808081, "accuracy_std_dev": 0.018027342695657245, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.044340436026936034, "avg_duration_per_task_ms": 217888.1734006734, "avg_output_tokens_per_task": 22075.030303030304, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.8072389999999999, "accuracy_std_dev": 0.0870155432092451, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.010984919191919193, "avg_duration_per_task_ms": 659527.0782828283, "avg_output_tokens_per_task": 25386.00420875421, "last_run_timestamp": "2026-10-03T17:34:53.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.8066155190839694, "accuracy_std_dev": 0.009958499999999981, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07586704071246819, "avg_duration_per_task_ms": 385930.98473282444, "avg_output_tokens_per_task": 32444.25445292621, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8065657, "accuracy_std_dev": 0.022453683925137975, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.3406123287878787, "avg_duration_per_task_ms": 355782.31666666665, "avg_output_tokens_per_task": 22552.313636363637, "last_run_timestamp": "2026-10-03T09:49:05.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8063972857142858, "accuracy_std_dev": 0.04410277557094591, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04038211976911977, "avg_duration_per_task_ms": 957664.380952381, "avg_output_tokens_per_task": 25231.96176046176, "last_run_timestamp": "2026-09-26T09:34:13.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.806210222222222, "accuracy_std_dev": 0.03697273211969953, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.25474633557800225, "avg_duration_per_task_ms": 231596.29349046017, "avg_output_tokens_per_task": 25466.63243546577, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.8058417989690723, "accuracy_std_dev": 0.019860476718151096, "run_count": 6, "total_questions": 1164, "avg_cost_per_task_usd": 0.1695892783505155, "avg_duration_per_task_ms": 882585.4982817869, "avg_output_tokens_per_task": 53988.948453608245, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.805555546875, "accuracy_std_dev": 0.01288499999999998, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.0362872109375, "avg_duration_per_task_ms": 729621.7864583334, "avg_output_tokens_per_task": 36271.864583333336, "last_run_timestamp": "2026-09-12T16:45:54.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.805068, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.07262640935672515, "avg_duration_per_task_ms": 352658, "avg_output_tokens_per_task": 15572.333333333334, "last_run_timestamp": "2026-07-18T08:30:38.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.8047647874762808, "accuracy_std_dev": 0.032856312422424996, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.016785316255534474, "avg_duration_per_task_ms": 574047.4073371284, "avg_output_tokens_per_task": 17963.88172043011, "last_run_timestamp": "2026-10-05T20:59:39.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8047135714285714, "accuracy_std_dev": 0.022568642992670425, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007838231601731601, "avg_duration_per_task_ms": 497728.7373737374, "avg_output_tokens_per_task": 18010.08152958153, "last_run_timestamp": "2026-10-04T00:40:00.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.8037036, "accuracy_std_dev": 0.10533181249480138, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.16579600808080808, "avg_duration_per_task_ms": 1110510.5636363637, "avg_output_tokens_per_task": 45531.156565656565, "last_run_timestamp": "2026-10-03T17:43:01.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.8034189230769232, "accuracy_std_dev": 0.024712500000000026, "run_count": 2, "total_questions": 195, "avg_cost_per_task_usd": 0.08983526153846154, "avg_duration_per_task_ms": 329694.73846153845, "avg_output_tokens_per_task": 37199.13846153846, "last_run_timestamp": "2026-10-03T16:53:50.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8032406250000002, "accuracy_std_dev": 0.023560394356512266, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3350150732323232, "avg_duration_per_task_ms": 352876.84217171714, "avg_output_tokens_per_task": 22221.229166666668, "last_run_timestamp": "2026-10-03T09:34:21.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8030233054141601, "accuracy_std_dev": 0.11015684719346006, "run_count": 11, "total_questions": 2161, "avg_cost_per_task_usd": 0.046368811198519204, "avg_duration_per_task_ms": 182261.37390097178, "avg_output_tokens_per_task": 22191.346136048127, "last_run_timestamp": "2026-10-03T10:14:18.163Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8020951111111111, "accuracy_std_dev": 0.027492606765554845, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04329651571268237, "avg_duration_per_task_ms": 84651.98765432098, "avg_output_tokens_per_task": 26957.542087542086, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.8016821288180611, "accuracy_std_dev": 0.026730709555864766, "run_count": 5, "total_questions": 753, "avg_cost_per_task_usd": 0.050311730411686593, "avg_duration_per_task_ms": 763057.3041168659, "avg_output_tokens_per_task": 21804.07835325365, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8016385562742562, "accuracy_std_dev": 0.007745278218856984, "run_count": 4, "total_questions": 773, "avg_cost_per_task_usd": 0.0594386532988357, "avg_duration_per_task_ms": 220979.64165588617, "avg_output_tokens_per_task": 17329.77360931436, "last_run_timestamp": "2026-09-26T08:53:08.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.8013615579868709, "accuracy_std_dev": 0.030272935016787066, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.2393555025528811, "avg_duration_per_task_ms": 897241.6353026987, "avg_output_tokens_per_task": 60323.876002917576, "last_run_timestamp": "2026-10-03T16:51:04.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.8011870934718102, "accuracy_std_dev": 0.030695975976321205, "run_count": 6, "total_questions": 674, "avg_cost_per_task_usd": 0.030323706231454008, "avg_duration_per_task_ms": 103892.11572700297, "avg_output_tokens_per_task": 31484.501483679524, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.8008437316455697, "accuracy_std_dev": 0.022930653266364256, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.028163265822784812, "avg_duration_per_task_ms": 479213.82784810127, "avg_output_tokens_per_task": 23504.80253164557, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.8007853333333332, "accuracy_std_dev": 0.008398773614178595, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0439676531986532, "avg_duration_per_task_ms": 201662.1632996633, "avg_output_tokens_per_task": 21888.12962962963, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.8006493194805195, "accuracy_std_dev": 0.04527692613517397, "run_count": 4, "total_questions": 770, "avg_cost_per_task_usd": 0.00850762987012987, "avg_duration_per_task_ms": 622509.1441558441, "avg_output_tokens_per_task": 19295.080519480518, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.7996635, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.046308881313131316, "avg_duration_per_task_ms": 53641.583333333336, "avg_output_tokens_per_task": 15349.742424242424, "last_run_timestamp": "2026-10-05T17:49:50.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7996631666666666, "accuracy_std_dev": 0.024569953550469384, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.045980551346801346, "avg_duration_per_task_ms": 194150.27525252526, "avg_output_tokens_per_task": 21808.128787878788, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7987012857142858, "accuracy_std_dev": 0.02392447384532228, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.028126163059163056, "avg_duration_per_task_ms": 291404.7186147186, "avg_output_tokens_per_task": 18604.992784992784, "last_run_timestamp": "2026-10-04T00:12:29.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7987011428571429, "accuracy_std_dev": 0.04484802990553949, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02699170274170274, "avg_duration_per_task_ms": 370961.9646464646, "avg_output_tokens_per_task": 25531.39466089466, "last_run_timestamp": "2026-10-03T19:56:06.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7979798571428571, "accuracy_std_dev": 0.006036422413236218, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.05221803823953824, "avg_duration_per_task_ms": 1137486.8282828282, "avg_output_tokens_per_task": 23798.989898989897, "last_run_timestamp": "2026-10-03T18:46:58.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7979796666666668, "accuracy_std_dev": 0.009816464683830371, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01347027946127946, "avg_duration_per_task_ms": 287878.5387205387, "avg_output_tokens_per_task": 14030.314814814816, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.797481, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.07670138953488372, "avg_duration_per_task_ms": 240811.48837209304, "avg_output_tokens_per_task": 21222.406976744187, "last_run_timestamp": "2026-07-18T08:37:00.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.7972860381679391, "accuracy_std_dev": 0.010877999999999999, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.01688720610687023, "avg_duration_per_task_ms": 167801.82442748093, "avg_output_tokens_per_task": 6563.4376590330785, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.7972582857142857, "accuracy_std_dev": 0.005891180301683102, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014391279220779221, "avg_duration_per_task_ms": 318122.0873015873, "avg_output_tokens_per_task": 14823.277056277057, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Fireworks", "accuracy": 0.797043, "accuracy_std_dev": null, "run_count": 1, "total_questions": 124, "avg_cost_per_task_usd": 0.11967770967741935, "avg_duration_per_task_ms": 375576.0080645161, "avg_output_tokens_per_task": 29097.58064516129, "last_run_timestamp": "2026-07-18T08:32:17.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7968575, "accuracy_std_dev": 0.009506569460992069, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015575840067340066, "avg_duration_per_task_ms": 306607.4048821549, "avg_output_tokens_per_task": 14907.355218855218, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7964375923664123, "accuracy_std_dev": 0.11378331267012533, "run_count": 4, "total_questions": 655, "avg_cost_per_task_usd": 0.011028116030534352, "avg_duration_per_task_ms": 270893.28854961833, "avg_output_tokens_per_task": 23218.567938931297, "last_run_timestamp": "2026-09-12T08:53:48.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7962963, "accuracy_std_dev": 0.10523964572541092, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0923363292929293, "avg_duration_per_task_ms": 515255.396969697, "avg_output_tokens_per_task": 34061.135353535356, "last_run_timestamp": "2026-10-03T17:43:17.222Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7956228, "accuracy_std_dev": 0.021522407276138952, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006844225252525253, "avg_duration_per_task_ms": 363476.91515151516, "avg_output_tokens_per_task": 16652.98888888889, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.7956228, "accuracy_std_dev": 0.009072301348610534, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.051206081818181814, "avg_duration_per_task_ms": 443200.41616161616, "avg_output_tokens_per_task": 24093.476767676766, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7947205871946083, "accuracy_std_dev": 0.04278072577075129, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.1992195315922494, "avg_duration_per_task_ms": 633124.2401010952, "avg_output_tokens_per_task": 47420.48188711036, "last_run_timestamp": "2026-10-03T17:09:08.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.794007441011236, "accuracy_std_dev": 0.03030512423976348, "run_count": 4, "total_questions": 356, "avg_cost_per_task_usd": 0.07293162640449438, "avg_duration_per_task_ms": 115799.61235955056, "avg_output_tokens_per_task": 31686.629213483146, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.7932827976391231, "accuracy_std_dev": 0.12137721453581074, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.12359699156829679, "avg_duration_per_task_ms": 670423.452782462, "avg_output_tokens_per_task": 39297.81534569983, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7929293333333333, "accuracy_std_dev": 0.008361060791284552, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01913193771043771, "avg_duration_per_task_ms": 454912.569023569, "avg_output_tokens_per_task": 25421.24579124579, "last_run_timestamp": "2026-10-03T18:54:19.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7924050379746834, "accuracy_std_dev": 0.007365480183486586, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.1897013772151899, "avg_duration_per_task_ms": 842584.7670886077, "avg_output_tokens_per_task": 47454.72573839663, "last_run_timestamp": "2026-10-03T17:06:15.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7923849784482757, "accuracy_std_dev": 0.02939258755007609, "run_count": 3, "total_questions": 464, "avg_cost_per_task_usd": 0.010662515086206897, "avg_duration_per_task_ms": 1267747.5711206896, "avg_output_tokens_per_task": 37807.1400862069, "last_run_timestamp": "2026-10-08T22:56:38.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7920900579268293, "accuracy_std_dev": 0.011512971285033247, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.01824498475609756, "avg_duration_per_task_ms": 181543.2469512195, "avg_output_tokens_per_task": 15051.83943089431, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7918084721906924, "accuracy_std_dev": 0.14984325869964388, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.016061496027241773, "avg_duration_per_task_ms": 528116.8059023836, "avg_output_tokens_per_task": 25582.743473325765, "last_run_timestamp": "2026-10-03T08:06:43.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7914971899592945, "accuracy_std_dev": 0.018448252406647506, "run_count": 8, "total_questions": 1474, "avg_cost_per_task_usd": 0.04180698643147897, "avg_duration_per_task_ms": 703517.4267299864, "avg_output_tokens_per_task": 19318.71234735414, "last_run_timestamp": "2026-10-08T23:28:00.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7912458333333334, "accuracy_std_dev": 0.01576267340287031, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01776989141414141, "avg_duration_per_task_ms": 61590.219696969696, "avg_output_tokens_per_task": 5770.594276094276, "last_run_timestamp": "2026-10-03T16:56:50.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.790816, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.013702010204081632, "avg_duration_per_task_ms": 552474.6071428572, "avg_output_tokens_per_task": 38202.7193877551, "last_run_timestamp": "2026-10-05T09:56:55.335Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7906953916083916, "accuracy_std_dev": 0.10313122487825346, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.07981278321678323, "avg_duration_per_task_ms": 812868.9388111888, "avg_output_tokens_per_task": 29607.183566433567, "last_run_timestamp": "2026-10-03T19:06:55.619Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7903146482300885, "accuracy_std_dev": 0.04010989916950373, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.14060938569321535, "avg_duration_per_task_ms": 1540325.2824483777, "avg_output_tokens_per_task": 46576.46017699115, "last_run_timestamp": "2026-10-03T16:58:27.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7900582903225808, "accuracy_std_dev": 0.015780706633417893, "run_count": 10, "total_questions": 1891, "avg_cost_per_task_usd": 0.004934497091485986, "avg_duration_per_task_ms": 1252981.0317292437, "avg_output_tokens_per_task": 32456.054997355895, "last_run_timestamp": "2026-10-09T00:11:24.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7891350392405064, "accuracy_std_dev": 0.020945155322890284, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.06166090379746835, "avg_duration_per_task_ms": 176165.68037974683, "avg_output_tokens_per_task": 19378.685443037975, "last_run_timestamp": "2026-10-03T10:40:10.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7890571, "accuracy_std_dev": 0.03611534435790416, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04233968282828283, "avg_duration_per_task_ms": 479457.0171717172, "avg_output_tokens_per_task": 42468.52626262626, "last_run_timestamp": "2026-10-03T17:31:01.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7880317272727273, "accuracy_std_dev": 0.018930562586323512, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.004806210284664829, "avg_duration_per_task_ms": 261371.71671258035, "avg_output_tokens_per_task": 31496.861340679523, "last_run_timestamp": "2026-10-03T17:02:47.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7865109761019649, "accuracy_std_dev": 0.015645091156333998, "run_count": 10, "total_questions": 1883, "avg_cost_per_task_usd": 0.008648618693574084, "avg_duration_per_task_ms": 393137.65427509294, "avg_output_tokens_per_task": 34261.50026553372, "last_run_timestamp": "2026-10-03T18:48:40.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7861111000000001, "accuracy_std_dev": 0.045978898789444694, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.17852181111111112, "avg_duration_per_task_ms": 793247.394949495, "avg_output_tokens_per_task": 59712.75959595959, "last_run_timestamp": "2026-10-03T17:12:34.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.7853823536121673, "accuracy_std_dev": 0.0034548367906458246, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.013724529784537389, "avg_duration_per_task_ms": 462082.7122940431, "avg_output_tokens_per_task": 14299.747782002534, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7835498571428571, "accuracy_std_dev": 0.056925857417292156, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.17942975324675325, "avg_duration_per_task_ms": 1414227.1096681096, "avg_output_tokens_per_task": 45704.106782106785, "last_run_timestamp": "2026-10-03T17:00:45.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7820105485714286, "accuracy_std_dev": 0.025091739481102143, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.055241464761904766, "avg_duration_per_task_ms": 515409.27365079365, "avg_output_tokens_per_task": 24015.726984126984, "last_run_timestamp": "2026-10-03T16:18:18.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.781145, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.020832949494949494, "avg_duration_per_task_ms": 266171.0505050505, "avg_output_tokens_per_task": 12380.515151515152, "last_run_timestamp": "2026-07-18T16:16:42.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7804232857142855, "accuracy_std_dev": 0.00964416607835736, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.021076865079365077, "avg_duration_per_task_ms": 281668.99494949495, "avg_output_tokens_per_task": 12536.775613275613, "last_run_timestamp": "2026-10-04T00:50:16.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7788999999999999, "accuracy_std_dev": 0.011690582106978247, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.018513430134680136, "avg_duration_per_task_ms": 636352.8888888889, "avg_output_tokens_per_task": 46067.3063973064, "last_run_timestamp": "2026-10-03T16:23:37.615Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7785069474835887, "accuracy_std_dev": 0.0632509601166138, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.21840516265499638, "avg_duration_per_task_ms": 918904.7702407002, "avg_output_tokens_per_task": 66679.34062727935, "last_run_timestamp": "2026-10-03T17:10:01.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7784511999999999, "accuracy_std_dev": 0.0206074889047647, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.05598715454545455, "avg_duration_per_task_ms": 643404.0878787879, "avg_output_tokens_per_task": 23899.834343434344, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7774409, "accuracy_std_dev": 0.02011474071644972, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.005779197474747475, "avg_duration_per_task_ms": 733463.898989899, "avg_output_tokens_per_task": 36574.07828282828, "last_run_timestamp": "2026-10-03T19:04:58.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7772166666666667, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013080006734006736, "avg_duration_per_task_ms": 67697.39225589225, "avg_output_tokens_per_task": 10341.513468013469, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7769442558139535, "accuracy_std_dev": 0.14033063886698557, "run_count": 7, "total_questions": 1333, "avg_cost_per_task_usd": 0.022806729182295572, "avg_duration_per_task_ms": 351644.4351087772, "avg_output_tokens_per_task": 23363.498124531132, "last_run_timestamp": "2026-10-03T19:19:20.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7763605976676385, "accuracy_std_dev": 0.09509307709474409, "run_count": 7, "total_questions": 1372, "avg_cost_per_task_usd": 0.05297574416909621, "avg_duration_per_task_ms": 1717526.1807580176, "avg_output_tokens_per_task": 18596.132653061224, "last_run_timestamp": "2026-10-05T23:17:57.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.77567325, "accuracy_std_dev": 0.010934638822910419, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013401992424242422, "avg_duration_per_task_ms": 67621.48106060606, "avg_output_tokens_per_task": 10598.125, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7756622557947019, "accuracy_std_dev": 0.011948399639177628, "run_count": 8, "total_questions": 1208, "avg_cost_per_task_usd": 0.052177753311258276, "avg_duration_per_task_ms": 995012.5024834437, "avg_output_tokens_per_task": 26633.495860927153, "last_run_timestamp": "2026-10-09T00:04:10.638Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7751544671163576, "accuracy_std_dev": 0.021435175618724375, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.022097016020236088, "avg_duration_per_task_ms": 821841.6846543002, "avg_output_tokens_per_task": 38407.37015177066, "last_run_timestamp": "2026-10-03T16:57:26.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7747440605802048, "accuracy_std_dev": 0.08568368177148243, "run_count": 6, "total_questions": 1172, "avg_cost_per_task_usd": 0.1293695656996587, "avg_duration_per_task_ms": 995226.8643344709, "avg_output_tokens_per_task": 37181.283276450515, "last_run_timestamp": "2026-10-03T16:47:49.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.774411, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007553676767676767, "avg_duration_per_task_ms": 145590.66666666666, "avg_output_tokens_per_task": 30019.929292929293, "last_run_timestamp": "2026-10-03T16:43:00.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7742798061728395, "accuracy_std_dev": 0.09894578471411179, "run_count": 9, "total_questions": 810, "avg_cost_per_task_usd": 0.003759838271604938, "avg_duration_per_task_ms": 204604.40740740742, "avg_output_tokens_per_task": 15966.761728395062, "last_run_timestamp": "2026-10-03T16:12:33.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7741139703903096, "accuracy_std_dev": 0.09718478875928833, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.004475604306864065, "avg_duration_per_task_ms": 312870.95827725437, "avg_output_tokens_per_task": 22926.07267833109, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7739002377332412, "accuracy_std_dev": 0.03919556245291041, "run_count": 9, "total_questions": 1447, "avg_cost_per_task_usd": 0.05904209744298548, "avg_duration_per_task_ms": 844538.2218382861, "avg_output_tokens_per_task": 50153.10711817553, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7730078333333333, "accuracy_std_dev": 0.02074231892706846, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015851826599326598, "avg_duration_per_task_ms": 424890.48232323234, "avg_output_tokens_per_task": 15979.028619528619, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7728757843137255, "accuracy_std_dev": 0.0424837728239854, "run_count": 5, "total_questions": 714, "avg_cost_per_task_usd": 0.13942839355742298, "avg_duration_per_task_ms": 1052327.7044817926, "avg_output_tokens_per_task": 40065.01820728291, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7716751249999998, "accuracy_std_dev": 0.010940746814517491, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0148653023989899, "avg_duration_per_task_ms": 144716.75757575757, "avg_output_tokens_per_task": 29722.781565656565, "last_run_timestamp": "2026-10-03T17:01:22.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7715112928416485, "accuracy_std_dev": 0.04364907130227424, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.13478754808387564, "avg_duration_per_task_ms": 1017266.7816341287, "avg_output_tokens_per_task": 61731.43745480839, "last_run_timestamp": "2026-10-03T17:38:07.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7713930343283582, "accuracy_std_dev": 0.03931269277500872, "run_count": 6, "total_questions": 670, "avg_cost_per_task_usd": 0.012700616417910445, "avg_duration_per_task_ms": 53684.10149253732, "avg_output_tokens_per_task": 16854.058208955223, "last_run_timestamp": "2026-10-03T17:06:37.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7711041709956711, "accuracy_std_dev": 0.019798063555812727, "run_count": 5, "total_questions": 924, "avg_cost_per_task_usd": 0.020921933982683983, "avg_duration_per_task_ms": 243233.94264069264, "avg_output_tokens_per_task": 27811.309523809523, "last_run_timestamp": "2026-10-03T16:58:21.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.770202, "accuracy_std_dev": 0.008625232557251217, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013833271043771043, "avg_duration_per_task_ms": 119359.86616161616, "avg_output_tokens_per_task": 10422.487373737375, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.770081857142857, "accuracy_std_dev": 0.03628441834581173, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.052005661616161616, "avg_duration_per_task_ms": 855189.3722943723, "avg_output_tokens_per_task": 23489.02380952381, "last_run_timestamp": "2026-10-03T18:36:41.004Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7700529885714287, "accuracy_std_dev": 0.007227262196667206, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.011466961904761905, "avg_duration_per_task_ms": 1005165.3326984127, "avg_output_tokens_per_task": 11934.448888888888, "last_run_timestamp": "2026-10-09T00:38:37.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7697014052863436, "accuracy_std_dev": 0.07503161624057866, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.22283792731277535, "avg_duration_per_task_ms": 1174636.9618208518, "avg_output_tokens_per_task": 71500.95080763583, "last_run_timestamp": "2026-10-03T16:59:22.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7693388734177216, "accuracy_std_dev": 0.018315649062573065, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.058541679324894506, "avg_duration_per_task_ms": 602157.2911392405, "avg_output_tokens_per_task": 26894.77299578059, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7689819826353422, "accuracy_std_dev": 0.01778415708685683, "run_count": 10, "total_questions": 1958, "avg_cost_per_task_usd": 0.008032920837589377, "avg_duration_per_task_ms": 455602.9090909091, "avg_output_tokens_per_task": 27245.283452502554, "last_run_timestamp": "2026-10-09T16:20:32.526Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7688279969604862, "accuracy_std_dev": 0.009398481531077219, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.019866108409321178, "avg_duration_per_task_ms": 104728.69604863222, "avg_output_tokens_per_task": 5941.306484295846, "last_run_timestamp": "2026-10-03T13:44:15.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.7685897384615384, "accuracy_std_dev": 0.017564000000000024, "run_count": 2, "total_questions": 260, "avg_cost_per_task_usd": 0.008261646153846154, "avg_duration_per_task_ms": 560681.2384615385, "avg_output_tokens_per_task": 27419.23076923077, "last_run_timestamp": "2026-10-03T16:10:46.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.768254, "accuracy_std_dev": null, "run_count": 1, "total_questions": 105, "avg_cost_per_task_usd": 0.04709281904761905, "avg_duration_per_task_ms": 1085275.6761904762, "avg_output_tokens_per_task": 21896.628571428573, "last_run_timestamp": "2026-07-18T16:43:10.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.767797, "accuracy_std_dev": 0.059792957643139584, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10487614862914862, "avg_duration_per_task_ms": 545212.5101010101, "avg_output_tokens_per_task": 31481.984126984127, "last_run_timestamp": "2026-10-03T16:59:42.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7676768, "accuracy_std_dev": 0.04513522126410817, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.002215911111111111, "avg_duration_per_task_ms": 785689.8707070707, "avg_output_tokens_per_task": 15243.155555555555, "last_run_timestamp": "2026-08-22T17:47:50.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7674760463576159, "accuracy_std_dev": 0.05316627521008299, "run_count": 7, "total_questions": 1359, "avg_cost_per_task_usd": 0.22231543340691684, "avg_duration_per_task_ms": 898904.8646063282, "avg_output_tokens_per_task": 67989.31420161884, "last_run_timestamp": "2026-10-03T16:12:13.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7673963333333335, "accuracy_std_dev": 0.012107287621740685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005097089225589225, "avg_duration_per_task_ms": 1089299.569023569, "avg_output_tokens_per_task": 28258.040404040403, "last_run_timestamp": "2026-10-09T03:53:21.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.765993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08016306060606061, "avg_duration_per_task_ms": 425226.22222222225, "avg_output_tokens_per_task": 69822.58585858585, "last_run_timestamp": "2026-07-18T08:29:23.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7647376591639872, "accuracy_std_dev": 0.13379493307180026, "run_count": 4, "total_questions": 622, "avg_cost_per_task_usd": 0.01636310610932476, "avg_duration_per_task_ms": 580282.652733119, "avg_output_tokens_per_task": 22094.519292604502, "last_run_timestamp": "2026-09-26T10:43:59.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7644980960665658, "accuracy_std_dev": 0.006467699256572579, "run_count": 7, "total_questions": 1322, "avg_cost_per_task_usd": 0.024349456127080185, "avg_duration_per_task_ms": 70318.21331316188, "avg_output_tokens_per_task": 25499.71180030257, "last_run_timestamp": "2026-10-03T17:11:16.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7643096666666667, "accuracy_std_dev": 0.01620599943093779, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.018970599326599325, "avg_duration_per_task_ms": 988034.0404040404, "avg_output_tokens_per_task": 47417.47222222222, "last_run_timestamp": "2026-10-03T16:16:35.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7639796666666666, "accuracy_std_dev": 0.03504930162627777, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.11775099782135079, "avg_duration_per_task_ms": 799196.2178649238, "avg_output_tokens_per_task": 45973.8779956427, "last_run_timestamp": "2026-10-03T16:58:32.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.763468, "accuracy_std_dev": 0.12590177777802292, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1458534191919192, "avg_duration_per_task_ms": 1775663.6776094276, "avg_output_tokens_per_task": 49845.639730639734, "last_run_timestamp": "2026-10-03T17:21:47.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.762821, "accuracy_std_dev": null, "run_count": 1, "total_questions": 78, "avg_cost_per_task_usd": 0.02737005128205128, "avg_duration_per_task_ms": 183098.3205128205, "avg_output_tokens_per_task": 13986.846153846154, "last_run_timestamp": "2026-10-08T18:00:04.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7627866948853615, "accuracy_std_dev": 0.026695964643035898, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.011795488536155203, "avg_duration_per_task_ms": 255692.54497354498, "avg_output_tokens_per_task": 43172.62081128748, "last_run_timestamp": "2026-10-05T10:43:46.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7613937607313196, "accuracy_std_dev": 0.03469481572741633, "run_count": 8, "total_questions": 1258, "avg_cost_per_task_usd": 0.04567820906200318, "avg_duration_per_task_ms": 263290.3680445151, "avg_output_tokens_per_task": 17206.77265500795, "last_run_timestamp": "2026-10-08T22:23:04.239Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.760943, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08179280303030302, "avg_duration_per_task_ms": 426340.89898989897, "avg_output_tokens_per_task": 70994.31818181818, "last_run_timestamp": "2026-07-18T08:26:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7605754720812183, "accuracy_std_dev": 0.010447569992586773, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0025078527918781725, "avg_duration_per_task_ms": 100160.06345177664, "avg_output_tokens_per_task": 31388.34771573604, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.7605443265306122, "accuracy_std_dev": 0.0343317834601635, "run_count": 3, "total_questions": 490, "avg_cost_per_task_usd": 0.0037374408163265305, "avg_duration_per_task_ms": 668153.0571428571, "avg_output_tokens_per_task": 14206.583673469388, "last_run_timestamp": "2026-08-08T11:42:26.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7597402857142859, "accuracy_std_dev": 0.006953716574903084, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01760951298701299, "avg_duration_per_task_ms": 82322.40548340548, "avg_output_tokens_per_task": 29108.308802308802, "last_run_timestamp": "2026-10-03T16:55:04.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7594230424028269, "accuracy_std_dev": 0.004131696745890251, "run_count": 3, "total_questions": 566, "avg_cost_per_task_usd": 0.017543240282685515, "avg_duration_per_task_ms": 1026275.9469964664, "avg_output_tokens_per_task": 53058.5777385159, "last_run_timestamp": "2026-10-09T04:51:43.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7590187142857142, "accuracy_std_dev": 0.03157783272538906, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.168772468975469, "avg_duration_per_task_ms": 557901.2611832612, "avg_output_tokens_per_task": 52553.58585858586, "last_run_timestamp": "2026-10-03T17:03:01.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7590111668383109, "accuracy_std_dev": 0.016611473377157127, "run_count": 5, "total_questions": 971, "avg_cost_per_task_usd": 0.004071515962924819, "avg_duration_per_task_ms": 4054.4541709577757, "avg_output_tokens_per_task": 2252.576725025747, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.758698, "accuracy_std_dev": 0.012009662443216293, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.056970961279461274, "avg_duration_per_task_ms": 99388.35016835017, "avg_output_tokens_per_task": 22699.07239057239, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7586280000000001, "accuracy_std_dev": 0.019486833125215618, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01705406755050505, "avg_duration_per_task_ms": 1145479.2203282828, "avg_output_tokens_per_task": 49997.517676767675, "last_run_timestamp": "2026-10-03T17:13:23.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.7568542857142856, "accuracy_std_dev": 0.010527309976767046, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.018623878066378066, "avg_duration_per_task_ms": 169039.05050505052, "avg_output_tokens_per_task": 30853.982683982686, "last_run_timestamp": "2026-10-03T16:34:22.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.7565450000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.020916916230366494, "avg_duration_per_task_ms": 124793.11518324607, "avg_output_tokens_per_task": 18115.47120418848, "last_run_timestamp": "2026-07-18T08:26:49.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7558922857142859, "accuracy_std_dev": 0.012501460717101212, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.018682439754689754, "avg_duration_per_task_ms": 169052.6507936508, "avg_output_tokens_per_task": 30918.866883116883, "last_run_timestamp": "2026-10-03T16:20:04.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.755892, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.035218818181818184, "avg_duration_per_task_ms": 133646.34848484848, "avg_output_tokens_per_task": 10567.11616161616, "last_run_timestamp": "2026-10-09T15:00:04.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7556517142857143, "accuracy_std_dev": 0.007605047378725084, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.022500825396825398, "avg_duration_per_task_ms": 97229.08225108225, "avg_output_tokens_per_task": 14839.808080808081, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7547599848156182, "accuracy_std_dev": 0.05601727479072497, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.0070590672451193055, "avg_duration_per_task_ms": 989833.9660159075, "avg_output_tokens_per_task": 18416.717281272595, "last_run_timestamp": "2026-10-04T00:33:38.079Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7547307336244543, "accuracy_std_dev": 0.013966941432525313, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.016579247452692868, "avg_duration_per_task_ms": 299714.7489082969, "avg_output_tokens_per_task": 28149.186317321688, "last_run_timestamp": "2026-10-03T17:05:00.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7537275714285715, "accuracy_std_dev": 0.04841524516353187, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.012184289321789322, "avg_duration_per_task_ms": 804903.6428571428, "avg_output_tokens_per_task": 36700.15007215007, "last_run_timestamp": "2026-10-03T10:19:08.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7524220592034444, "accuracy_std_dev": 0.028639192749568526, "run_count": 7, "total_questions": 929, "avg_cost_per_task_usd": 0.05453179763186222, "avg_duration_per_task_ms": 423386.32615715824, "avg_output_tokens_per_task": 15836.034445640473, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7509868781725888, "accuracy_std_dev": 0.06676355715258636, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.2357046700507614, "avg_duration_per_task_ms": 1233772.4551607445, "avg_output_tokens_per_task": 75729.32741116751, "last_run_timestamp": "2026-10-03T16:45:32.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7503429835390947, "accuracy_std_dev": 0.031096298525708804, "run_count": 7, "total_questions": 1215, "avg_cost_per_task_usd": 0.007341672427983538, "avg_duration_per_task_ms": 1064868.6510288066, "avg_output_tokens_per_task": 18926.89547325103, "last_run_timestamp": "2026-10-09T00:38:01.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.749368625, "accuracy_std_dev": 0.008643231874384451, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.013557867424242424, "avg_duration_per_task_ms": 96888.43560606061, "avg_output_tokens_per_task": 14901.809343434343, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.749158, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016407893939393937, "avg_duration_per_task_ms": 1120280.1313131314, "avg_output_tokens_per_task": 58254.84848484849, "last_run_timestamp": "2026-10-03T11:59:33.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.7481442884615385, "accuracy_std_dev": 0.0227081467548543, "run_count": 5, "total_questions": 988, "avg_cost_per_task_usd": 0.020536520242914982, "avg_duration_per_task_ms": 151472.34615384616, "avg_output_tokens_per_task": 11878.324898785426, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7478116, "accuracy_std_dev": 0.02151720496347049, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08178134747474748, "avg_duration_per_task_ms": 885057.2202020203, "avg_output_tokens_per_task": 23371.92828282828, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08105034848484848, "avg_duration_per_task_ms": 422601, "avg_output_tokens_per_task": 70367.68181818182, "last_run_timestamp": "2026-07-18T08:27:39.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016666974747474746, "avg_duration_per_task_ms": 1024552.0252525252, "avg_output_tokens_per_task": 59525.38383838384, "last_run_timestamp": "2026-10-03T09:35:36.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7470487959527824, "accuracy_std_dev": 0.08033451694532995, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.005070625070264193, "avg_duration_per_task_ms": 656330.5952782462, "avg_output_tokens_per_task": 33326.38673412029, "last_run_timestamp": "2026-09-26T18:04:35.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.746633, "accuracy_std_dev": 0.017677, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017912724747474747, "avg_duration_per_task_ms": 70698.09595959596, "avg_output_tokens_per_task": 6119.75, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7465302921840761, "accuracy_std_dev": 0.11934042810107118, "run_count": 7, "total_questions": 1369, "avg_cost_per_task_usd": 0.16837692549306063, "avg_duration_per_task_ms": 1144637.61577794, "avg_output_tokens_per_task": 50455.26296566837, "last_run_timestamp": "2026-10-03T18:23:36.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.74642275, "accuracy_std_dev": 0.008945525344411051, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01488374558080808, "avg_duration_per_task_ms": 107961.75883838384, "avg_output_tokens_per_task": 14400.517676767677, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7461931421319796, "accuracy_std_dev": 0.031200414988782142, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.004617404399323182, "avg_duration_per_task_ms": 792703.1945854484, "avg_output_tokens_per_task": 27906.86040609137, "last_run_timestamp": "2026-10-03T17:02:01.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7459782222222222, "accuracy_std_dev": 0.009438398095931553, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006886203142536476, "avg_duration_per_task_ms": 163678.69079685747, "avg_output_tokens_per_task": 8003.4517396184065, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7457915, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014859042929292928, "avg_duration_per_task_ms": 987926.6818181818, "avg_output_tokens_per_task": 52717.79797979798, "last_run_timestamp": "2026-10-03T10:50:00.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7457912857142857, "accuracy_std_dev": 0.038516164256857485, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04952885064935065, "avg_duration_per_task_ms": 952735.3326118327, "avg_output_tokens_per_task": 26072.224386724385, "last_run_timestamp": "2026-10-03T17:12:39.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7443558079681274, "accuracy_std_dev": 0.10340629541961245, "run_count": 7, "total_questions": 1255, "avg_cost_per_task_usd": 0.033485443027888445, "avg_duration_per_task_ms": 150958.26772908366, "avg_output_tokens_per_task": 29080.37370517928, "last_run_timestamp": "2026-10-03T17:57:43.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.744108, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012609747474747475, "avg_duration_per_task_ms": 233838.9696969697, "avg_output_tokens_per_task": 62878.5101010101, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7436087853982302, "accuracy_std_dev": 0.07436948975876863, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.13460593289085546, "avg_duration_per_task_ms": 1037566.0044247787, "avg_output_tokens_per_task": 57351.7883480826, "last_run_timestamp": "2026-10-03T16:56:39.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7434143503039515, "accuracy_std_dev": 0.004376274946591358, "run_count": 7, "total_questions": 1316, "avg_cost_per_task_usd": 0.009106649696048632, "avg_duration_per_task_ms": 140201.952887538, "avg_output_tokens_per_task": 11539.308510638299, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7433862857142857, "accuracy_std_dev": 0.006785370391285875, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005382690476190476, "avg_duration_per_task_ms": 149045.53823953823, "avg_output_tokens_per_task": 8700.336940836942, "last_run_timestamp": "2026-10-03T17:05:50.656Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.743266, "accuracy_std_dev": 0.09234344800932008, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.029721953914141414, "avg_duration_per_task_ms": 1313239.2840909092, "avg_output_tokens_per_task": 18789.802398989897, "last_run_timestamp": "2026-10-05T21:43:30.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.743145857142857, "accuracy_std_dev": 0.011088924595914524, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.023026453102453104, "avg_duration_per_task_ms": 99585.75685425685, "avg_output_tokens_per_task": 15242.642857142857, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.74242425, "accuracy_std_dev": 0.08095717220967824, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017491213383838383, "avg_duration_per_task_ms": 2247201.621212121, "avg_output_tokens_per_task": 17091.73232323232, "last_run_timestamp": "2026-10-04T01:00:47.859Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7415824999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.019016893939393938, "avg_duration_per_task_ms": 33403.05808080808, "avg_output_tokens_per_task": 6109.055555555556, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.740741, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02792859090909091, "avg_duration_per_task_ms": 682232.3585858586, "avg_output_tokens_per_task": 23364.934343434343, "last_run_timestamp": "2026-07-18T09:15:42.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7406781050847459, "accuracy_std_dev": 0.10340659429589846, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.006412678531073445, "avg_duration_per_task_ms": 534746.5333333333, "avg_output_tokens_per_task": 27485.332768361583, "last_run_timestamp": "2026-10-03T10:03:14.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.739899, "accuracy_std_dev": 0.010812416138403098, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0048791994949494944, "avg_duration_per_task_ms": 108739.66098484848, "avg_output_tokens_per_task": 7956.729797979798, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7395381428571429, "accuracy_std_dev": 0.07707797586939116, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0066781443001443, "avg_duration_per_task_ms": 95940.86147186147, "avg_output_tokens_per_task": 16425.329725829724, "last_run_timestamp": "2026-10-03T16:54:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.7392292244897959, "accuracy_std_dev": 0.110097, "run_count": 2, "total_questions": 294, "avg_cost_per_task_usd": 0.03822905782312925, "avg_duration_per_task_ms": 1403982.7517006802, "avg_output_tokens_per_task": 27800.656462585033, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7387387252252252, "accuracy_std_dev": 0.018499983466208795, "run_count": 4, "total_questions": 444, "avg_cost_per_task_usd": 0.01493145045045045, "avg_duration_per_task_ms": 79529.70945945945, "avg_output_tokens_per_task": 8581.265765765766, "last_run_timestamp": "2026-10-04T00:00:36.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7387387027027028, "accuracy_std_dev": 0.06584221235841033, "run_count": 6, "total_questions": 1184, "avg_cost_per_task_usd": 0.010049336993243244, "avg_duration_per_task_ms": 495836.71199324325, "avg_output_tokens_per_task": 34244.69847972973, "last_run_timestamp": "2026-10-03T17:30:20.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7385762857142857, "accuracy_std_dev": 0.1483657159518949, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.030072527417027418, "avg_duration_per_task_ms": 138129.5901875902, "avg_output_tokens_per_task": 23037.04256854257, "last_run_timestamp": "2026-10-03T16:48:25.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.738309, "accuracy_std_dev": 0.008136215650479735, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.059655349046015715, "avg_duration_per_task_ms": 125601.6632996633, "avg_output_tokens_per_task": 23853.966891133558, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7378293623304071, "accuracy_std_dev": 0.0050744218168684305, "run_count": 7, "total_questions": 1253, "avg_cost_per_task_usd": 0.0064119712689545095, "avg_duration_per_task_ms": 226069.1827613727, "avg_output_tokens_per_task": 7932.774142059058, "last_run_timestamp": "2026-10-03T16:52:01.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.736652142857143, "accuracy_std_dev": 0.05013373986342806, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.016218962481962484, "avg_duration_per_task_ms": 475113.9826839827, "avg_output_tokens_per_task": 15698.901154401154, "last_run_timestamp": "2026-10-04T00:29:23.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.736111, "accuracy_std_dev": null, "run_count": 1, "total_questions": 168, "avg_cost_per_task_usd": 0.05987852380952381, "avg_duration_per_task_ms": 1235915.3511904762, "avg_output_tokens_per_task": 62102.07738095238, "last_run_timestamp": "2026-07-18T08:29:05.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7344877142857142, "accuracy_std_dev": 0.06178190798217846, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011764064935064936, "avg_duration_per_task_ms": 95375.22294372294, "avg_output_tokens_per_task": 5009.694083694084, "last_run_timestamp": "2026-10-03T10:46:59.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7322156158536586, "accuracy_std_dev": 0.0628101300320259, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.015606842479674796, "avg_duration_per_task_ms": 1212509.0548780488, "avg_output_tokens_per_task": 39797.57621951219, "last_run_timestamp": "2026-10-03T17:20:21.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7316240615384616, "accuracy_std_dev": 0.012678999999999996, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.014848610256410256, "avg_duration_per_task_ms": 978212.5923076923, "avg_output_tokens_per_task": 52658.97692307692, "last_run_timestamp": "2026-10-03T10:14:05.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7313943448275863, "accuracy_std_dev": 0.013515629737139094, "run_count": 29, "total_questions": 5742, "avg_cost_per_task_usd": 0.2044269764890282, "avg_duration_per_task_ms": 243437.65552072448, "avg_output_tokens_per_task": 38184.69731800766, "last_run_timestamp": "2026-10-03T09:50:25.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7300516810519218, "accuracy_std_dev": 0.016762033534018325, "run_count": 15, "total_questions": 2966, "avg_cost_per_task_usd": 0.0067854190829399865, "avg_duration_per_task_ms": 104234.09912339851, "avg_output_tokens_per_task": 7076.775792312879, "last_run_timestamp": "2026-10-03T16:40:17.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7297978999999999, "accuracy_std_dev": 0.016326424363282986, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.18559751616161616, "avg_duration_per_task_ms": 291435.0106060606, "avg_output_tokens_per_task": 36955.371717171714, "last_run_timestamp": "2026-10-03T09:40:02.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7286196, "accuracy_std_dev": 0.010123398502479267, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.19021829797979797, "avg_duration_per_task_ms": 241955.48484848486, "avg_output_tokens_per_task": 37852.89646464647, "last_run_timestamp": "2026-10-03T09:25:15.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7277538571428572, "accuracy_std_dev": 0.020296230068157783, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005383534632034633, "avg_duration_per_task_ms": 215362.52525252526, "avg_output_tokens_per_task": 7827.647907647907, "last_run_timestamp": "2026-10-03T17:25:00.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7272725714285714, "accuracy_std_dev": 0.015455781912624552, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004581955266955267, "avg_duration_per_task_ms": 160113.02813852814, "avg_output_tokens_per_task": 7774.143578643579, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.726431, "accuracy_std_dev": 0.012012832596852407, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00610699074074074, "avg_duration_per_task_ms": 365363.45791245793, "avg_output_tokens_per_task": 8100.830808080808, "last_run_timestamp": "2026-10-03T17:45:28.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7261503333333333, "accuracy_std_dev": 0.004418546090690373, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.059926168350168355, "avg_duration_per_task_ms": 123042.70033670033, "avg_output_tokens_per_task": 23940.483164983165, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.725799625, "accuracy_std_dev": 0.008125736488735932, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004432963383838384, "avg_duration_per_task_ms": 640133.6079545454, "avg_output_tokens_per_task": 7937.186237373738, "last_run_timestamp": "2026-10-09T00:02:28.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7255892857142857, "accuracy_std_dev": 0.030568975597183873, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007296898268398268, "avg_duration_per_task_ms": 250965.08297258298, "avg_output_tokens_per_task": 16490.68903318903, "last_run_timestamp": "2026-10-03T17:08:10.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7234887554404145, "accuracy_std_dev": 0.009381891562472875, "run_count": 10, "total_questions": 1930, "avg_cost_per_task_usd": 0.18696787461139894, "avg_duration_per_task_ms": 349087.42279792746, "avg_output_tokens_per_task": 37203.45544041451, "last_run_timestamp": "2026-10-03T09:27:16.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7225028666666667, "accuracy_std_dev": 0.012643758973589382, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.1973207321548822, "avg_duration_per_task_ms": 263893.9067340067, "avg_output_tokens_per_task": 36800.280303030304, "last_run_timestamp": "2026-10-03T09:27:45.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7222221428571429, "accuracy_std_dev": 0.020500479544416028, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006640360028860029, "avg_duration_per_task_ms": 334191.47041847045, "avg_output_tokens_per_task": 26367.450937950936, "last_run_timestamp": "2026-10-03T17:40:21.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7220491744548286, "accuracy_std_dev": 0.030848909598882115, "run_count": 5, "total_questions": 963, "avg_cost_per_task_usd": 0.16807563966770508, "avg_duration_per_task_ms": 1503192.1879543094, "avg_output_tokens_per_task": 41124.30737279335, "last_run_timestamp": "2026-08-22T16:49:30.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7211000000000001, "accuracy_std_dev": 0.05129065401220774, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015817102693602692, "avg_duration_per_task_ms": 896061.7971380472, "avg_output_tokens_per_task": 43246.910774410775, "last_run_timestamp": "2026-10-03T18:22:40.169Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.71923592494929, "accuracy_std_dev": 0.031713523289599956, "run_count": 5, "total_questions": 986, "avg_cost_per_task_usd": 0.01125353448275862, "avg_duration_per_task_ms": 528851.3123732251, "avg_output_tokens_per_task": 56006.78093306288, "last_run_timestamp": "2026-10-03T09:18:14.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7184345000000002, "accuracy_std_dev": 0.00575544305418794, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006676434343434343, "avg_duration_per_task_ms": 1198975.5896464647, "avg_output_tokens_per_task": 54074.54671717172, "last_run_timestamp": "2026-09-26T17:35:27.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.7184345000000001, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005931222222222222, "avg_duration_per_task_ms": 277051.3181818182, "avg_output_tokens_per_task": 7213.616161616162, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7171302234809476, "accuracy_std_dev": 0.05915089253318348, "run_count": 6, "total_questions": 971, "avg_cost_per_task_usd": 0.035917489186405756, "avg_duration_per_task_ms": 751381.8146240988, "avg_output_tokens_per_task": 38973.5623069001, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7170402190889371, "accuracy_std_dev": 0.1172294876991454, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.004009827187274042, "avg_duration_per_task_ms": 280047.9363702097, "avg_output_tokens_per_task": 10071.284887924801, "last_run_timestamp": "2026-10-04T00:01:24.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.715769, "accuracy_std_dev": 0.047798870551509894, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00680639898989899, "avg_duration_per_task_ms": 127099.18602693603, "avg_output_tokens_per_task": 7554.406565656565, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.715488, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0037085252525252527, "avg_duration_per_task_ms": 839996.601010101, "avg_output_tokens_per_task": 28561.196969696968, "last_run_timestamp": "2026-07-18T16:28:34.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.715488, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018400479797979798, "avg_duration_per_task_ms": 232169.87878787878, "avg_output_tokens_per_task": 65366.41414141414, "last_run_timestamp": "2026-10-07T21:00:03.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7133716737044147, "accuracy_std_dev": 0.07918700388461619, "run_count": 3, "total_questions": 521, "avg_cost_per_task_usd": 0.07439642610364683, "avg_duration_per_task_ms": 652167.8637236084, "avg_output_tokens_per_task": 28734.940499040305, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7118807142857142, "accuracy_std_dev": 0.008414141170235586, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011084775613275614, "avg_duration_per_task_ms": 94339.78354978355, "avg_output_tokens_per_task": 10485.185425685426, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.71170025, "accuracy_std_dev": 0.018456257615982147, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0649322058080808, "avg_duration_per_task_ms": 1066951.6603535353, "avg_output_tokens_per_task": 21770.531565656565, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7113995714285716, "accuracy_std_dev": 0.010488730194930244, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007985493506493507, "avg_duration_per_task_ms": 77746.15223665224, "avg_output_tokens_per_task": 10141.235930735931, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7109188571428572, "accuracy_std_dev": 0.009120000930898651, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008814419191919191, "avg_duration_per_task_ms": 134404.79509379508, "avg_output_tokens_per_task": 5713.798701298701, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7106782857142858, "accuracy_std_dev": 0.008509694097478088, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015819831890331888, "avg_duration_per_task_ms": 94996.398989899, "avg_output_tokens_per_task": 10465.311688311689, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.70883299486008, "accuracy_std_dev": 0.036241367349453366, "run_count": 9, "total_questions": 1751, "avg_cost_per_task_usd": 0.010877087378640777, "avg_duration_per_task_ms": 281941.7778412336, "avg_output_tokens_per_task": 47824.729868646486, "last_run_timestamp": "2026-10-03T09:36:23.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.7083335000000002, "accuracy_std_dev": 0.053758401224831845, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05601492550505051, "avg_duration_per_task_ms": 1569531.130050505, "avg_output_tokens_per_task": 33285.93813131313, "last_run_timestamp": "2026-10-04T00:00:36.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7080899875730994, "accuracy_std_dev": 0.01572159476124274, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.010683734649122806, "avg_duration_per_task_ms": 102665.67105263157, "avg_output_tokens_per_task": 9845.541666666666, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7078187849794239, "accuracy_std_dev": 0.11990455329935001, "run_count": 6, "total_questions": 972, "avg_cost_per_task_usd": 0.004627639917695473, "avg_duration_per_task_ms": 565545.3467078189, "avg_output_tokens_per_task": 25055.12037037037, "last_run_timestamp": "2026-10-05T10:41:53.582Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7051465714285715, "accuracy_std_dev": 0.008927717479163138, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006021121212121212, "avg_duration_per_task_ms": 109220.25974025975, "avg_output_tokens_per_task": 5588.1832611832615, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.70328275, "accuracy_std_dev": 0.02153875190644758, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0019388005050505052, "avg_duration_per_task_ms": 636092.8409090909, "avg_output_tokens_per_task": 14000.35101010101, "last_run_timestamp": "2026-09-05T08:28:07.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7024411666666667, "accuracy_std_dev": 0.14218853509503576, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013173934343434343, "avg_duration_per_task_ms": 786438.8409090909, "avg_output_tokens_per_task": 27569.127104377105, "last_run_timestamp": "2026-09-22T02:20:03.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7021902115384616, "accuracy_std_dev": 0.04770844742400843, "run_count": 16, "total_questions": 3120, "avg_cost_per_task_usd": 0.00835266217948718, "avg_duration_per_task_ms": 486695.7522435897, "avg_output_tokens_per_task": 25245.021153846155, "last_run_timestamp": "2026-10-09T00:36:36.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.701832040429564, "accuracy_std_dev": 0.009254626092359155, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.010471590018951359, "avg_duration_per_task_ms": 144460.25710675932, "avg_output_tokens_per_task": 9453.23373341756, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7017798571428572, "accuracy_std_dev": 0.013719199460490942, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011180141414141413, "avg_duration_per_task_ms": 199426.63852813852, "avg_output_tokens_per_task": 5820.556277056277, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.7010584285714286, "accuracy_std_dev": 0.024870525026666448, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011166472582972582, "avg_duration_per_task_ms": 61418.856421356424, "avg_output_tokens_per_task": 9244.735930735931, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6952191613545817, "accuracy_std_dev": 0.030707686936010004, "run_count": 5, "total_questions": 502, "avg_cost_per_task_usd": 0.009013709163346614, "avg_duration_per_task_ms": 48627.035856573704, "avg_output_tokens_per_task": 5832.633466135459, "last_run_timestamp": "2026-10-04T00:12:48.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.6928812857142858, "accuracy_std_dev": 0.0184605601038251, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.009158278499278501, "avg_duration_per_task_ms": 267700.48124098126, "avg_output_tokens_per_task": 6402.472582972583, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6926927837837837, "accuracy_std_dev": 0.02593753677273029, "run_count": 7, "total_questions": 1332, "avg_cost_per_task_usd": 0.009898264264264264, "avg_duration_per_task_ms": 376272.99399399396, "avg_output_tokens_per_task": 45270.37087087087, "last_run_timestamp": "2026-10-03T09:53:13.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.692129625, "accuracy_std_dev": 0.010010686751385976, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005500493055555555, "avg_duration_per_task_ms": 380445.4021464646, "avg_output_tokens_per_task": 6066.023358585859, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Wafer", "accuracy": 0.691919, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01248349494949495, "avg_duration_per_task_ms": 400806.7474747475, "avg_output_tokens_per_task": 31001.929292929293, "last_run_timestamp": "2026-07-18T08:04:42.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6917878385615913, "accuracy_std_dev": 0.2742550167259424, "run_count": 7, "total_questions": 1307, "avg_cost_per_task_usd": 0.02420585692425402, "avg_duration_per_task_ms": 312930.753634277, "avg_output_tokens_per_task": 14561.572302983932, "last_run_timestamp": "2026-10-04T00:21:14.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.685185, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007298616161616161, "avg_duration_per_task_ms": 578052.9191919192, "avg_output_tokens_per_task": 17923.358585858587, "last_run_timestamp": "2026-07-18T08:44:31.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.685185, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06544819696969698, "avg_duration_per_task_ms": 945099.904040404, "avg_output_tokens_per_task": 33006.14646464647, "last_run_timestamp": "2026-07-18T16:15:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.68308075, "accuracy_std_dev": 0.050431518420403505, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007981098484848485, "avg_duration_per_task_ms": 281140.702020202, "avg_output_tokens_per_task": 41483.12815656565, "last_run_timestamp": "2026-10-08T22:19:02.584Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.6809765714285714, "accuracy_std_dev": 0.0445115533536989, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014742932178932178, "avg_duration_per_task_ms": 268442.60533910536, "avg_output_tokens_per_task": 7991.235930735931, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6782486254237288, "accuracy_std_dev": 0.021428719439237503, "run_count": 6, "total_questions": 590, "avg_cost_per_task_usd": 0.004103340677966102, "avg_duration_per_task_ms": 494419.3559322034, "avg_output_tokens_per_task": 27366.218644067798, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6779694919852034, "accuracy_std_dev": 0.0848101181666433, "run_count": 5, "total_questions": 811, "avg_cost_per_task_usd": 0.007035279901356351, "avg_duration_per_task_ms": 476700.88409371144, "avg_output_tokens_per_task": 34252.66584463625, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.676784615514334, "accuracy_std_dev": 0.11674700703943836, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.05877558516020236, "avg_duration_per_task_ms": 524798.5919055649, "avg_output_tokens_per_task": 24486.29679595278, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6750842, "accuracy_std_dev": 0.023641210809939493, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005652539393939394, "avg_duration_per_task_ms": 500294.62828282826, "avg_output_tokens_per_task": 16033.422222222222, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.6737955840707965, "accuracy_std_dev": 0.07392353888102138, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.1320932337758112, "avg_duration_per_task_ms": 1052059.2595870206, "avg_output_tokens_per_task": 71927.51327433628, "last_run_timestamp": "2026-10-03T16:56:56.694Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6728395, "accuracy_std_dev": 0.030761988442177545, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005704560606060605, "avg_duration_per_task_ms": 413488.24074074073, "avg_output_tokens_per_task": 15636.112794612794, "last_run_timestamp": "2026-10-03T17:31:07.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6679221148775893, "accuracy_std_dev": 0.015946572790414858, "run_count": 5, "total_questions": 531, "avg_cost_per_task_usd": 0.004098755178907722, "avg_duration_per_task_ms": 736779.5291902071, "avg_output_tokens_per_task": 40851.82109227872, "last_run_timestamp": "2026-10-09T16:25:16.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.6678159586206897, "accuracy_std_dev": 0.07976717100897782, "run_count": 7, "total_questions": 1015, "avg_cost_per_task_usd": 0.04199740492610838, "avg_duration_per_task_ms": 264794.87290640397, "avg_output_tokens_per_task": 15915.662068965517, "last_run_timestamp": "2026-10-08T22:52:37.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.6673423259988418, "accuracy_std_dev": 0.09700477765553617, "run_count": 10, "total_questions": 1727, "avg_cost_per_task_usd": 0.030584045165026053, "avg_duration_per_task_ms": 798364.2964678634, "avg_output_tokens_per_task": 32108.440069484655, "last_run_timestamp": "2026-10-09T00:06:52.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009070929292929292, "avg_duration_per_task_ms": 492518.0909090909, "avg_output_tokens_per_task": 69555.08585858585, "last_run_timestamp": "2026-07-18T17:04:25.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.002231131313131313, "avg_duration_per_task_ms": 1386615.121212121, "avg_output_tokens_per_task": 14198.39898989899, "last_run_timestamp": "2026-07-18T09:58:41.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6654557258916777, "accuracy_std_dev": 0.015299762836703567, "run_count": 8, "total_questions": 1514, "avg_cost_per_task_usd": 0.007279266182298546, "avg_duration_per_task_ms": 504110.9504623514, "avg_output_tokens_per_task": 51807.61756935271, "last_run_timestamp": "2026-10-08T23:53:57.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.6654041250000001, "accuracy_std_dev": 0.0318059948022283, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0025243055555555557, "avg_duration_per_task_ms": 211186.44002525252, "avg_output_tokens_per_task": 5302.534090909091, "last_run_timestamp": "2026-10-03T16:12:07.496Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6647026666666667, "accuracy_std_dev": 0.07359682177392055, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006663367845117845, "avg_duration_per_task_ms": 681258.2592592592, "avg_output_tokens_per_task": 22537.15824915825, "last_run_timestamp": "2026-10-03T17:57:14.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6632997142857142, "accuracy_std_dev": 0.009351528808768375, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0021804473304473304, "avg_duration_per_task_ms": 240867.02741702742, "avg_output_tokens_per_task": 3011.5793650793653, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.663019, "accuracy_std_dev": 0.007921062218330754, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0027949452861952863, "avg_duration_per_task_ms": 83787.81902356903, "avg_output_tokens_per_task": 2948.4612794612794, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6616161818181817, "accuracy_std_dev": 0.013544295262167041, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.005160062442607897, "avg_duration_per_task_ms": 106935.65518824609, "avg_output_tokens_per_task": 9720.085399449035, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6607744999999999, "accuracy_std_dev": 0.011572094721786573, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015565636363636365, "avg_duration_per_task_ms": 14159.460858585859, "avg_output_tokens_per_task": 1751.3977272727273, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6602454107648725, "accuracy_std_dev": 0.21094196359873227, "run_count": 9, "total_questions": 1765, "avg_cost_per_task_usd": 0.014101193201133146, "avg_duration_per_task_ms": 357051.99943342776, "avg_output_tokens_per_task": 17061.87252124646, "last_run_timestamp": "2026-09-26T08:53:03.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6596062028241334, "accuracy_std_dev": 0.016648273731831805, "run_count": 4, "total_questions": 779, "avg_cost_per_task_usd": 0.011759379974326058, "avg_duration_per_task_ms": 1526360.2888318356, "avg_output_tokens_per_task": 50819.251604621306, "last_run_timestamp": "2026-10-03T17:41:49.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.6595962, "accuracy_std_dev": 0.011187342962473291, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0024944898989898985, "avg_duration_per_task_ms": 89627.26666666666, "avg_output_tokens_per_task": 2311.8535353535353, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6594517142857144, "accuracy_std_dev": 0.023871300387789566, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.027211155122655123, "avg_duration_per_task_ms": 890187.8643578644, "avg_output_tokens_per_task": 18800.659451659452, "last_run_timestamp": "2026-10-03T16:57:14.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.656205, "accuracy_std_dev": 0.016336828302947918, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.008925360028860031, "avg_duration_per_task_ms": 319523.6370851371, "avg_output_tokens_per_task": 59199.669191919194, "last_run_timestamp": "2026-10-03T17:56:49.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6549602541636494, "accuracy_std_dev": 0.09036662854873644, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.011515717595944969, "avg_duration_per_task_ms": 1066625.0246198408, "avg_output_tokens_per_task": 33708.20130340333, "last_run_timestamp": "2026-10-08T22:36:32.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6512783352272727, "accuracy_std_dev": 0.07301920016707937, "run_count": 5, "total_questions": 704, "avg_cost_per_task_usd": 0.06126106249999999, "avg_duration_per_task_ms": 1570715.0482954546, "avg_output_tokens_per_task": 22881.873579545456, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.650363823580786, "accuracy_std_dev": 0.007103203700131033, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.0036315545851528385, "avg_duration_per_task_ms": 58503.03493449782, "avg_output_tokens_per_task": 2202.661135371179, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6495510000000001, "accuracy_std_dev": 0.00792134559276387, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0032256414141414147, "avg_duration_per_task_ms": 40985.67760942761, "avg_output_tokens_per_task": 3060.60101010101, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.649551, "accuracy_std_dev": 0.009444634067377443, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002492604377104377, "avg_duration_per_task_ms": 61525.246632996634, "avg_output_tokens_per_task": 2691.0799663299663, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.011195005920300148, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004075954545454546, "avg_duration_per_task_ms": 28668.510101010103, "avg_output_tokens_per_task": 2354.649831649832, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6487794999999998, "accuracy_std_dev": 0.12844600199597497, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6039836092171716, "avg_duration_per_task_ms": 625325.3693181818, "avg_output_tokens_per_task": 40216.142676767675, "last_run_timestamp": "2026-10-03T10:37:00.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6476671428571429, "accuracy_std_dev": 0.01801337100543905, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007287767676767677, "avg_duration_per_task_ms": 701126.4617604618, "avg_output_tokens_per_task": 56538.52525252525, "last_run_timestamp": "2026-10-03T18:53:13.795Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6473065000000001, "accuracy_std_dev": 0.009707729665065862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00413114393939394, "avg_duration_per_task_ms": 27377.72601010101, "avg_output_tokens_per_task": 2388.9305555555557, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.64646475, "accuracy_std_dev": 0.02185317805692116, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007503073232323232, "avg_duration_per_task_ms": 814987.6553030303, "avg_output_tokens_per_task": 52529.47348484849, "last_run_timestamp": "2026-10-03T20:08:16.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6464646666666667, "accuracy_std_dev": 0.013537798302858865, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015368191919191919, "avg_duration_per_task_ms": 14060.520202020201, "avg_output_tokens_per_task": 1727.3535353535353, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6457914, "accuracy_std_dev": 0.017547304854022453, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007042358080808081, "avg_duration_per_task_ms": 238513.37575757576, "avg_output_tokens_per_task": 11322.903535353535, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6447479677419355, "accuracy_std_dev": 0.02407747186802543, "run_count": 3, "total_questions": 403, "avg_cost_per_task_usd": 0.003165789081885856, "avg_duration_per_task_ms": 173618.99503722086, "avg_output_tokens_per_task": 10324.468982630273, "last_run_timestamp": "2026-10-09T03:20:25.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6445006666666666, "accuracy_std_dev": 0.013998443861928202, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002421077441077441, "avg_duration_per_task_ms": 66682.89730639731, "avg_output_tokens_per_task": 2836.0850168350166, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.643659, "accuracy_std_dev": 0.015752669318774726, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015761657407407408, "avg_duration_per_task_ms": 18961.11531986532, "avg_output_tokens_per_task": 1683.8872053872053, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.643098, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018197469696969695, "avg_duration_per_task_ms": 1435371.2424242424, "avg_output_tokens_per_task": 32217.19191919192, "last_run_timestamp": "2026-07-18T17:46:02.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.6428570571428571, "accuracy_std_dev": 0.0682565, "run_count": 2, "total_questions": 175, "avg_cost_per_task_usd": 0.09669735428571428, "avg_duration_per_task_ms": 1267820.5257142857, "avg_output_tokens_per_task": 34805.99428571429, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6428055813449023, "accuracy_std_dev": 0.011146034629373067, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.0026296478669558935, "avg_duration_per_task_ms": 140713.84381778742, "avg_output_tokens_per_task": 9405.384671005062, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6413729024390243, "accuracy_std_dev": 0.03637250000000003, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.027402341463414633, "avg_duration_per_task_ms": 991300.8970189702, "avg_output_tokens_per_task": 32627.39024390244, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6406363579175706, "accuracy_std_dev": 0.016263383886158854, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.002550553145336226, "avg_duration_per_task_ms": 225461.96384671004, "avg_output_tokens_per_task": 8275.081706435285, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6405723333333334, "accuracy_std_dev": 0.0030349496792460334, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004210166666666667, "avg_duration_per_task_ms": 30431.28367003367, "avg_output_tokens_per_task": 2308.9638047138046, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.6402916666666668, "accuracy_std_dev": 0.11880589124562244, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0025993900112233448, "avg_duration_per_task_ms": 225334.6874298541, "avg_output_tokens_per_task": 6052.3058361391695, "last_run_timestamp": "2026-10-05T20:06:05.844Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.6402118571428571, "accuracy_std_dev": 0.01348951470511788, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004506121212121212, "avg_duration_per_task_ms": 72990.16883116883, "avg_output_tokens_per_task": 8472.02886002886, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.639316, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.0024375846153846152, "avg_duration_per_task_ms": 97724.68205128206, "avg_output_tokens_per_task": 7853.553846153846, "last_run_timestamp": "2026-07-18T08:33:31.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6384837407407408, "accuracy_std_dev": 0.1301056437821581, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.6548667364384586, "avg_duration_per_task_ms": 493199.0927796483, "avg_output_tokens_per_task": 40750.34081556304, "last_run_timestamp": "2026-10-03T10:04:02.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6380471428571428, "accuracy_std_dev": 0.01709744716473838, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.009828660894660893, "avg_duration_per_task_ms": 659565.3961038961, "avg_output_tokens_per_task": 50620.7886002886, "last_run_timestamp": "2026-10-03T20:22:45.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6371981347826087, "accuracy_std_dev": 0.014429669830892581, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.0032497253623188405, "avg_duration_per_task_ms": 218812.71086956523, "avg_output_tokens_per_task": 10563.355072463768, "last_run_timestamp": "2026-10-03T16:41:22.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6368643521865888, "accuracy_std_dev": 0.13454240742991402, "run_count": 26, "total_questions": 5145, "avg_cost_per_task_usd": 0.6621504369290573, "avg_duration_per_task_ms": 598729.5551020408, "avg_output_tokens_per_task": 41288.28415937804, "last_run_timestamp": "2026-10-03T10:34:21.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.636738, "accuracy_std_dev": 0.1325038277837034, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.6152635606060606, "avg_duration_per_task_ms": 631219.6868686868, "avg_output_tokens_per_task": 40865.88888888889, "last_run_timestamp": "2026-10-03T09:11:52.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6361772567678425, "accuracy_std_dev": 0.013327375155318935, "run_count": 7, "total_questions": 1219, "avg_cost_per_task_usd": 0.007504426579163248, "avg_duration_per_task_ms": 35368.206726825265, "avg_output_tokens_per_task": 12428.807219031993, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6353917965162311, "accuracy_std_dev": 0.014982452781401947, "run_count": 7, "total_questions": 1263, "avg_cost_per_task_usd": 0.016417965162311955, "avg_duration_per_task_ms": 97650.78147268409, "avg_output_tokens_per_task": 54500.377672209026, "last_run_timestamp": "2026-10-03T17:21:59.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6352981489141675, "accuracy_std_dev": 0.02103570346434842, "run_count": 5, "total_questions": 967, "avg_cost_per_task_usd": 0.006571570837642193, "avg_duration_per_task_ms": 890942.791106515, "avg_output_tokens_per_task": 36274.74457083764, "last_run_timestamp": "2026-10-03T08:12:46.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6352257765567765, "accuracy_std_dev": 0.022072905759475025, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.013415796703296703, "avg_duration_per_task_ms": 548616.1373626373, "avg_output_tokens_per_task": 46655.3021978022, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6340491250000001, "accuracy_std_dev": 0.012397945832248788, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0016212228535353534, "avg_duration_per_task_ms": 88876.8813131313, "avg_output_tokens_per_task": 8195.30366161616, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.633745, "accuracy_std_dev": 0.1400560587510904, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.6191078804713805, "avg_duration_per_task_ms": 634861.303030303, "avg_output_tokens_per_task": 41204.60381593715, "last_run_timestamp": "2026-10-03T10:40:39.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.632856543563069, "accuracy_std_dev": 0.17446938590265632, "run_count": 4, "total_questions": 769, "avg_cost_per_task_usd": 0.06782110143042913, "avg_duration_per_task_ms": 1537627.533159948, "avg_output_tokens_per_task": 34138.841352405725, "last_run_timestamp": "2026-08-22T17:02:13.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6328437272727273, "accuracy_std_dev": 0.03335048624993999, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.006873024334251608, "avg_duration_per_task_ms": 112649.99265381084, "avg_output_tokens_per_task": 9586.507805325988, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6325426993295513, "accuracy_std_dev": 0.010534965676735725, "run_count": 10, "total_questions": 1939, "avg_cost_per_task_usd": 0.01766013512119649, "avg_duration_per_task_ms": 478451.96183599794, "avg_output_tokens_per_task": 88282.73388344508, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.632154875, "accuracy_std_dev": 0.01720972334494007, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0017519513888888888, "avg_duration_per_task_ms": 214138.922979798, "avg_output_tokens_per_task": 9606.703282828283, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.6319606293706294, "accuracy_std_dev": 0.11679624770601185, "run_count": 7, "total_questions": 1287, "avg_cost_per_task_usd": 0.09059485703185705, "avg_duration_per_task_ms": 737977.4195804195, "avg_output_tokens_per_task": 28229.06216006216, "last_run_timestamp": "2026-10-03T16:55:14.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.63118499609375, "accuracy_std_dev": 0.12107794090401254, "run_count": 3, "total_questions": 512, "avg_cost_per_task_usd": 0.00688423046875, "avg_duration_per_task_ms": 160958.9140625, "avg_output_tokens_per_task": 8535.359375, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.6297015953307392, "accuracy_std_dev": 0.01592646788319292, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.023244770428015562, "avg_duration_per_task_ms": 1042943.9273670558, "avg_output_tokens_per_task": 96652.03437094683, "last_run_timestamp": "2026-10-03T17:27:37.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6293489999999999, "accuracy_std_dev": 0.008199388066597833, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0011419814814814814, "avg_duration_per_task_ms": 334863.3063973064, "avg_output_tokens_per_task": 11634.42760942761, "last_run_timestamp": "2026-10-03T17:38:50.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6285795064570466, "accuracy_std_dev": 0.020919173358428876, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01718929421673217, "avg_duration_per_task_ms": 631165.4817518248, "avg_output_tokens_per_task": 83912.1010668164, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.626263, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012936333333333333, "avg_duration_per_task_ms": 385474.66161616164, "avg_output_tokens_per_task": 30893.848484848484, "last_run_timestamp": "2026-07-18T16:20:00.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.6261091711026616, "accuracy_std_dev": 0.05551308764606775, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.0414238669201521, "avg_duration_per_task_ms": 392853.54372623574, "avg_output_tokens_per_task": 20615.842839036755, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.623263890625, "accuracy_std_dev": 0.046540499999999985, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.03095198958333333, "avg_duration_per_task_ms": 834783.7890625, "avg_output_tokens_per_task": 38538.341145833336, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.623188452685422, "accuracy_std_dev": 0.0187657309831867, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.014858657289002557, "avg_duration_per_task_ms": 724190.4782608695, "avg_output_tokens_per_task": 15645.849104859335, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008206156565656566, "avg_duration_per_task_ms": 339475.95454545453, "avg_output_tokens_per_task": 15670.888888888889, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.619708208029197, "accuracy_std_dev": 0.029586940351224878, "run_count": 7, "total_questions": 1370, "avg_cost_per_task_usd": 0.06579531313868613, "avg_duration_per_task_ms": 1037486.9379562044, "avg_output_tokens_per_task": 26177.450364963504, "last_run_timestamp": "2026-10-04T00:28:53.518Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6148083751080381, "accuracy_std_dev": 0.23157299340777357, "run_count": 7, "total_questions": 1157, "avg_cost_per_task_usd": 0.005702518582541054, "avg_duration_per_task_ms": 156483.37078651684, "avg_output_tokens_per_task": 9454.971477960242, "last_run_timestamp": "2026-10-03T16:57:17.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "NextBit", "accuracy": 0.614478, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009234040404040404, "avg_duration_per_task_ms": 990972.4696969697, "avg_output_tokens_per_task": 16689.560606060608, "last_run_timestamp": "2026-07-18T08:03:56.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6132539090909092, "accuracy_std_dev": 0.1299958112139524, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.6145130344352616, "avg_duration_per_task_ms": 628297.9467401286, "avg_output_tokens_per_task": 40850.65381083563, "last_run_timestamp": "2026-10-03T10:13:27.517Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6128760749682338, "accuracy_std_dev": 0.007225322047320813, "run_count": 4, "total_questions": 787, "avg_cost_per_task_usd": 0.0033133367217280814, "avg_duration_per_task_ms": 404332.3011435832, "avg_output_tokens_per_task": 13398.312579415502, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6108306666666666, "accuracy_std_dev": 0.019131244624668038, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005598868686868688, "avg_duration_per_task_ms": 615283.2525252525, "avg_output_tokens_per_task": 18654.404882154882, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6102353048048047, "accuracy_std_dev": 0.004626267235105652, "run_count": 7, "total_questions": 1332, "avg_cost_per_task_usd": 0.004039552552552552, "avg_duration_per_task_ms": 36383.835585585584, "avg_output_tokens_per_task": 2146.3363363363364, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6096681428571429, "accuracy_std_dev": 0.010277069224089844, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0036427301587301586, "avg_duration_per_task_ms": 34466.44155844156, "avg_output_tokens_per_task": 2150.969696969697, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6096471504551988, "accuracy_std_dev": 0.016445311419549056, "run_count": 11, "total_questions": 2087, "avg_cost_per_task_usd": 0.008342372783900336, "avg_duration_per_task_ms": 542298.5462386201, "avg_output_tokens_per_task": 13701.597987541927, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6084389949367088, "accuracy_std_dev": 0.010283734794972733, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.008408201687763715, "avg_duration_per_task_ms": 279347.2447257384, "avg_output_tokens_per_task": 17442.14936708861, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6077443333333332, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0077788232323232325, "avg_duration_per_task_ms": 287812.98148148146, "avg_output_tokens_per_task": 15362.217171717171, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004114661616161616, "avg_duration_per_task_ms": 51474.35353535353, "avg_output_tokens_per_task": 2201.479797979798, "last_run_timestamp": "2026-07-18T08:26:39.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.6060605, "accuracy_std_dev": 0.001683499999999949, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008235411616161616, "avg_duration_per_task_ms": 152037.7323232323, "avg_output_tokens_per_task": 15653.954545454546, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.605239731707317, "accuracy_std_dev": 0.09170649999999997, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.03298509756097561, "avg_duration_per_task_ms": 869238.1111111111, "avg_output_tokens_per_task": 34362.55826558266, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6046178571428571, "accuracy_std_dev": 0.006995035043894045, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0034120865800865803, "avg_duration_per_task_ms": 53141.78787878788, "avg_output_tokens_per_task": 2114.74531024531, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6036557142857142, "accuracy_std_dev": 0.0048341465405498454, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011036557720057719, "avg_duration_per_task_ms": 33894.917748917745, "avg_output_tokens_per_task": 2102.8852813852814, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6026559059760956, "accuracy_std_dev": 0.018857781098506346, "run_count": 7, "total_questions": 1255, "avg_cost_per_task_usd": 0.002205658167330677, "avg_duration_per_task_ms": 47812.13147410358, "avg_output_tokens_per_task": 2066.2470119521913, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.6002888571428572, "accuracy_std_dev": 0.011234048798548233, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.003266399711399712, "avg_duration_per_task_ms": 60828.217893217894, "avg_output_tokens_per_task": 2084.8766233766232, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.599327, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014991068181818182, "avg_duration_per_task_ms": 284433.01515151514, "avg_output_tokens_per_task": 12302.075757575758, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.5980126829268292, "accuracy_std_dev": 0.061359, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.03206006233062331, "avg_duration_per_task_ms": 831219.6937669377, "avg_output_tokens_per_task": 40208.840108401084, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5973626666666666, "accuracy_std_dev": 0.014397638598819675, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003072445286195286, "avg_duration_per_task_ms": 655562.0867003367, "avg_output_tokens_per_task": 13595.005050505051, "last_run_timestamp": "2026-10-07T13:51:26.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.5968335954198474, "accuracy_std_dev": 0.034825528389147715, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.024714866836301953, "avg_duration_per_task_ms": 1074343.121289228, "avg_output_tokens_per_task": 64653.8074639525, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.591751, "accuracy_std_dev": 0.006627755628164546, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017204867003367003, "avg_duration_per_task_ms": 118615.4696969697, "avg_output_tokens_per_task": 42873.59259259259, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5878479469525959, "accuracy_std_dev": 0.06702947204028986, "run_count": 10, "total_questions": 1772, "avg_cost_per_task_usd": 0.009048836343115122, "avg_duration_per_task_ms": 528814.486455982, "avg_output_tokens_per_task": 45050.80643340858, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5860806989010988, "accuracy_std_dev": 0.014672256934467207, "run_count": 7, "total_questions": 1365, "avg_cost_per_task_usd": 0.006546478388278388, "avg_duration_per_task_ms": 728895.4901098901, "avg_output_tokens_per_task": 43384.46666666667, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5836942857142858, "accuracy_std_dev": 0.01603982892422384, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.010374448051948051, "avg_duration_per_task_ms": 11818.474025974027, "avg_output_tokens_per_task": 13378.126262626263, "last_run_timestamp": "2026-10-03T17:02:12.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.5767120715096482, "accuracy_std_dev": 0.20359371905450435, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.023737154370034052, "avg_duration_per_task_ms": 751908.8337116913, "avg_output_tokens_per_task": 33435.202610669694, "last_run_timestamp": "2026-10-03T17:20:14.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5750844, "accuracy_std_dev": 0.02978589572666902, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0254069303030303, "avg_duration_per_task_ms": 1275373.0656565656, "avg_output_tokens_per_task": 63407.47878787879, "last_run_timestamp": "2026-08-30T00:49:28.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5717949931623931, "accuracy_std_dev": 0.029316541569280166, "run_count": 6, "total_questions": 585, "avg_cost_per_task_usd": 0.032133635897435904, "avg_duration_per_task_ms": 57054.20170940171, "avg_output_tokens_per_task": 6867.617094017094, "last_run_timestamp": "2026-10-04T00:00:34.891Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.5676551155419224, "accuracy_std_dev": 0.08425995197346976, "run_count": 7, "total_questions": 978, "avg_cost_per_task_usd": 0.09743748875255624, "avg_duration_per_task_ms": 1096781.8711656441, "avg_output_tokens_per_task": 27754.531697341514, "last_run_timestamp": "2026-10-08T23:22:48.508Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.564814935483871, "accuracy_std_dev": 0.015030477400564811, "run_count": 3, "total_questions": 558, "avg_cost_per_task_usd": 0.0051593978494623655, "avg_duration_per_task_ms": 615212.1326164874, "avg_output_tokens_per_task": 14268.974910394265, "last_run_timestamp": "2026-10-05T18:22:40.566Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.5633876206896551, "accuracy_std_dev": 0.09194318097738406, "run_count": 5, "total_questions": 986, "avg_cost_per_task_usd": 0.029487100405679513, "avg_duration_per_task_ms": 2640133.111561866, "avg_output_tokens_per_task": 23360.7738336714, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5630111428571428, "accuracy_std_dev": 0.01687958484110801, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004603796536796536, "avg_duration_per_task_ms": 419928.0497835498, "avg_output_tokens_per_task": 16214.744588744588, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.56229, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0034265555555555555, "avg_duration_per_task_ms": 3024088.404040404, "avg_output_tokens_per_task": 30654.242424242424, "last_run_timestamp": "2026-07-18T16:00:09.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5602958771331057, "accuracy_std_dev": 0.05661860675431709, "run_count": 5, "total_questions": 586, "avg_cost_per_task_usd": 0.05058588907849829, "avg_duration_per_task_ms": 611241.4914675768, "avg_output_tokens_per_task": 24217.037542662118, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5505053333333333, "accuracy_std_dev": 0.016890890559378122, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.017397131313131312, "avg_duration_per_task_ms": 129348.1271043771, "avg_output_tokens_per_task": 43400.656565656565, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5488216666666668, "accuracy_std_dev": 0.006298902038362641, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0171891531986532, "avg_duration_per_task_ms": 121387.12121212122, "avg_output_tokens_per_task": 42892.78114478115, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.535985, "accuracy_std_dev": 0.011835000612589788, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0018682651515151514, "avg_duration_per_task_ms": 92043.92676767676, "avg_output_tokens_per_task": 1839.9911616161617, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.5353534999999999, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005200381313131313, "avg_duration_per_task_ms": 113966.90151515152, "avg_output_tokens_per_task": 11220.5, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.5331086666666667, "accuracy_std_dev": 0.05754091454106565, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005119237373737374, "avg_duration_per_task_ms": 101325.84511784512, "avg_output_tokens_per_task": 5420.193602693603, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.5310693945086705, "accuracy_std_dev": 0.0817773924709448, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.009087861994219654, "avg_duration_per_task_ms": 353941.95809248555, "avg_output_tokens_per_task": 26784.09176300578, "last_run_timestamp": "2026-10-03T16:13:54.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5282142748691099, "accuracy_std_dev": 0.013706854874680634, "run_count": 6, "total_questions": 1146, "avg_cost_per_task_usd": 0.01651366666666667, "avg_duration_per_task_ms": 725814.7731239093, "avg_output_tokens_per_task": 41825.98516579407, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.522126, "accuracy_std_dev": 0.02474951208754282, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01514130735930736, "avg_duration_per_task_ms": 13491.472582972583, "avg_output_tokens_per_task": 1320.8239538239538, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.517469628909552, "accuracy_std_dev": 0.07510589055164418, "run_count": 6, "total_questions": 1183, "avg_cost_per_task_usd": 0.003801546914623838, "avg_duration_per_task_ms": 137539.94759087067, "avg_output_tokens_per_task": 4071.0912933220625, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.513468, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006161969696969697, "avg_duration_per_task_ms": 165410.29292929292, "avg_output_tokens_per_task": 12691.05303030303, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5080647741935485, "accuracy_std_dev": 0.12443414088451235, "run_count": 6, "total_questions": 1116, "avg_cost_per_task_usd": 0.004283293010752688, "avg_duration_per_task_ms": 107626.95967741935, "avg_output_tokens_per_task": 3879.1989247311826, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.5056116666666668, "accuracy_std_dev": 0.007570922855827376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012942734006734005, "avg_duration_per_task_ms": 12612.308080808081, "avg_output_tokens_per_task": 1100.3198653198654, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5, "accuracy_std_dev": 0.015120700182200542, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008362104377104377, "avg_duration_per_task_ms": 19874.548821548822, "avg_output_tokens_per_task": 1898.5521885521885, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.49764320000000006, "accuracy_std_dev": 0.01852187333290023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0008495191919191919, "avg_duration_per_task_ms": 13033.880808080808, "avg_output_tokens_per_task": 1851.6353535353535, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4966332500000001, "accuracy_std_dev": 0.014773079898501197, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008230315656565656, "avg_duration_per_task_ms": 17739.39898989899, "avg_output_tokens_per_task": 1863.909090909091, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.49601803668941985, "accuracy_std_dev": 0.017030085894772614, "run_count": 6, "total_questions": 1172, "avg_cost_per_task_usd": 0.01113635409556314, "avg_duration_per_task_ms": 463652.90699658706, "avg_output_tokens_per_task": 27776.748293515357, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.49537025, "accuracy_std_dev": 0.016425006010272865, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.002352652146464646, "avg_duration_per_task_ms": 10902.190025252525, "avg_output_tokens_per_task": 2331.179292929293, "last_run_timestamp": "2026-10-03T16:58:35.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.4939061636197441, "accuracy_std_dev": 0.024075339672231702, "run_count": 6, "total_questions": 1094, "avg_cost_per_task_usd": 0.010896170018281536, "avg_duration_per_task_ms": 477480.3711151737, "avg_output_tokens_per_task": 26830.819012797074, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.48621612499999994, "accuracy_std_dev": 0.005119178558067202, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.002366635732323232, "avg_duration_per_task_ms": 13903.847853535353, "avg_output_tokens_per_task": 2226.974116161616, "last_run_timestamp": "2026-10-03T16:13:49.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.4840632105511069, "accuracy_std_dev": 0.017868444314808295, "run_count": 11, "total_questions": 2123, "avg_cost_per_task_usd": 0.0011087908619877532, "avg_duration_per_task_ms": 61751.02025435704, "avg_output_tokens_per_task": 2146.873763542157, "last_run_timestamp": "2026-10-09T15:26:28.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.48063949999999994, "accuracy_std_dev": 0.003857437653417109, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04852650126262626, "avg_duration_per_task_ms": 45991.458333333336, "avg_output_tokens_per_task": 17009.132575757576, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.4772725, "accuracy_std_dev": 0.0074500120973056095, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002467788720538721, "avg_duration_per_task_ms": 42647.26262626263, "avg_output_tokens_per_task": 2666.861952861953, "last_run_timestamp": "2026-10-03T16:39:56.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.47667114285714296, "accuracy_std_dev": 0.012154133775663925, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0013214949494949496, "avg_duration_per_task_ms": 47661.26911976912, "avg_output_tokens_per_task": 2366.4862914862915, "last_run_timestamp": "2026-10-03T16:14:03.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.4766039334298119, "accuracy_std_dev": 0.01707390743987169, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.0013226222865412446, "avg_duration_per_task_ms": 98211.54196816208, "avg_output_tokens_per_task": 3034.6222865412446, "last_run_timestamp": "2026-10-03T17:16:32.671Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.474536875, "accuracy_std_dev": 0.01241241161537012, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.003145674873737374, "avg_duration_per_task_ms": 53376.7619949495, "avg_output_tokens_per_task": 2218.6376262626263, "last_run_timestamp": "2026-10-03T17:41:38.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.4741861666666667, "accuracy_std_dev": 0.012808772416026279, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010600345117845118, "avg_duration_per_task_ms": 199985.06313131313, "avg_output_tokens_per_task": 2395.604377104377, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.4713801428571428, "accuracy_std_dev": 0.009733517737746226, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0013883506493506494, "avg_duration_per_task_ms": 69013.97546897546, "avg_output_tokens_per_task": 2273.031746031746, "last_run_timestamp": "2026-10-03T16:57:23.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0022648939393939394, "avg_duration_per_task_ms": 33028.21717171717, "avg_output_tokens_per_task": 2380.9646464646466, "last_run_timestamp": "2026-07-18T16:13:31.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.46742483824768327, "accuracy_std_dev": 0.006948085005956674, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.002137747262005055, "avg_duration_per_task_ms": 34994.02274641955, "avg_output_tokens_per_task": 2587.217354675653, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46106887500000004, "accuracy_std_dev": 0.01782538514617215, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0007930915404040404, "avg_duration_per_task_ms": 157279.59027777778, "avg_output_tokens_per_task": 2227.837121212121, "last_run_timestamp": "2026-10-08T22:58:25.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.45862900000000006, "accuracy_std_dev": null, "run_count": 1, "total_questions": 141, "avg_cost_per_task_usd": 0.0011888297872340425, "avg_duration_per_task_ms": 278701.0070921986, "avg_output_tokens_per_task": 7719.879432624113, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4320986666666667, "accuracy_std_dev": 0.00781631474687543, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010060690235690235, "avg_duration_per_task_ms": 20918.510101010103, "avg_output_tokens_per_task": 1483.5286195286196, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4315373333333334, "accuracy_std_dev": 0.010588221658469791, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010723358585858589, "avg_duration_per_task_ms": 21765.176767676767, "avg_output_tokens_per_task": 1505.858585858586, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4305555, "accuracy_std_dev": 0.007469952225416168, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010262436868686867, "avg_duration_per_task_ms": 17602.7702020202, "avg_output_tokens_per_task": 1517.9292929292928, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.40740725, "accuracy_std_dev": 0.3006428663691981, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017353968434343434, "avg_duration_per_task_ms": 189056.44444444444, "avg_output_tokens_per_task": 19318.219696969696, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.3994878450704225, "accuracy_std_dev": 0.0901063557941392, "run_count": 4, "total_questions": 781, "avg_cost_per_task_usd": 0.016465559539052498, "avg_duration_per_task_ms": 3369255.4609475033, "avg_output_tokens_per_task": 16587.451984635085, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.38720499999999997, "accuracy_std_dev": 0.010647388881786938, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00015740101010101008, "avg_duration_per_task_ms": 6591.242424242424, "avg_output_tokens_per_task": 937.0363636363636, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.382155, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.05791383333333333, "avg_duration_per_task_ms": 1031309.4747474748, "avg_output_tokens_per_task": 22994.131313131315, "last_run_timestamp": "2026-10-03T19:39:23.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.37878799999999996, "accuracy_std_dev": 0.01683499999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000344469696969697, "avg_duration_per_task_ms": 46431.74494949495, "avg_output_tokens_per_task": 2036.358585858586, "last_run_timestamp": "2026-10-07T13:35:26.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.350168, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011657070707070707, "avg_duration_per_task_ms": 7587.606060606061, "avg_output_tokens_per_task": 355.9040404040404, "last_run_timestamp": "2026-07-18T09:09:45.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.3483847621145374, "accuracy_std_dev": 0.08501017676557487, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.00338636490455213, "avg_duration_per_task_ms": 34858.14243759178, "avg_output_tokens_per_task": 1419.5543318649045, "last_run_timestamp": "2026-10-03T16:11:17.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Novita", "accuracy": 0.344383, "accuracy_std_dev": null, "run_count": 1, "total_questions": 181, "avg_cost_per_task_usd": 0.00011365193370165745, "avg_duration_per_task_ms": 62171.872928176796, "avg_output_tokens_per_task": 485.27624309392263, "last_run_timestamp": "2026-07-18T08:00:15.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.338384, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00013078787878787878, "avg_duration_per_task_ms": 16627.883838383837, "avg_output_tokens_per_task": 667.8888888888889, "last_run_timestamp": "2026-07-18T09:08:52.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DeepInfra", "accuracy": 0.301347, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00002756060606060606, "avg_duration_per_task_ms": 15713.474747474747, "avg_output_tokens_per_task": 431.8787878787879, "last_run_timestamp": "2026-07-18T08:00:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DekaLLM", "accuracy": 0.289562, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00002257070707070707, "avg_duration_per_task_ms": 19325.131313131315, "avg_output_tokens_per_task": 285.6515151515151, "last_run_timestamp": "2026-07-18T08:04:03.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2891415, "accuracy_std_dev": 0.003232916137174003, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006112626262626263, "avg_duration_per_task_ms": 10017.040404040405, "avg_output_tokens_per_task": 577.7083333333334, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.2886002857142857, "accuracy_std_dev": 0.01680762263901442, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0015827532467532469, "avg_duration_per_task_ms": 97706.50937950939, "avg_output_tokens_per_task": 20607.09163059163, "last_run_timestamp": "2026-10-03T17:14:12.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.2783791428571428, "accuracy_std_dev": 0.011859313909672988, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0022877604617604613, "avg_duration_per_task_ms": 48962.33477633478, "avg_output_tokens_per_task": 28673.977633477632, "last_run_timestamp": "2026-10-03T16:55:38.344Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.27641208165057063, "accuracy_std_dev": 0.013856526257206988, "run_count": 6, "total_questions": 1139, "avg_cost_per_task_usd": 0.007524540825285337, "avg_duration_per_task_ms": 300907.412642669, "avg_output_tokens_per_task": 33503.4363476734, "last_run_timestamp": "2026-10-03T17:43:23.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2739361560283688, "accuracy_std_dev": 0.008153796102974918, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.00006610283687943263, "avg_duration_per_task_ms": 8879.929078014184, "avg_output_tokens_per_task": 551.2074468085107, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.273148, "accuracy_std_dev": 0.006113657170630353, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006268181818181819, "avg_duration_per_task_ms": 9553.181818181818, "avg_output_tokens_per_task": 586.439393939394, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.27244666666666667, "accuracy_std_dev": 0.06424850635012633, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008843218013468015, "avg_duration_per_task_ms": 589547.7979797979, "avg_output_tokens_per_task": 21876.559764309764, "last_run_timestamp": "2026-10-03T17:50:31.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.2700681612244898, "accuracy_std_dev": 0.016752437809465226, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.00005762346938775511, "avg_duration_per_task_ms": 9734.870408163266, "avg_output_tokens_per_task": 576.8010204081633, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.2668783085714286, "accuracy_std_dev": 0.01699007316742264, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.00036382412698412705, "avg_duration_per_task_ms": 401076.15555555554, "avg_output_tokens_per_task": 8879.231746031746, "last_run_timestamp": "2026-10-08T23:14:52.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.25484025, "accuracy_std_dev": 0.015150020327956655, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0003991950757575757, "avg_duration_per_task_ms": 68227.3188131313, "avg_output_tokens_per_task": 7660.273358585859, "last_run_timestamp": "2026-10-03T16:12:07.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.254209, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.001291050505050505, "avg_duration_per_task_ms": 40150.21212121212, "avg_output_tokens_per_task": 5062.732323232323, "last_run_timestamp": "2026-07-18T08:26:34.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.23148166666666667, "accuracy_std_dev": 0.10429433789947032, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0013719949494949493, "avg_duration_per_task_ms": 502712.0462962963, "avg_output_tokens_per_task": 4456.498316498317, "last_run_timestamp": "2026-08-29T17:16:14.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.1910775, "accuracy_std_dev": 0.11363615432122239, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0014224511784511782, "avg_duration_per_task_ms": 46218.05892255892, "avg_output_tokens_per_task": 1225.9587542087543, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.060606, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.000040161616161616165, "avg_duration_per_task_ms": 3671.1060606060605, "avg_output_tokens_per_task": 93.72222222222223, "last_run_timestamp": "2026-08-01T08:05:34.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.002245, "accuracy_std_dev": 0.0007933738084913065, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003512888888888889, "avg_duration_per_task_ms": 175891.9090909091, "avg_output_tokens_per_task": 3312.4141414141413, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034544065656565656, "avg_duration_per_task_ms": 164367.952020202, "avg_output_tokens_per_task": 3253.931818181818, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.822917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.0087319375, "avg_duration_per_task_ms": 31283.875, "avg_output_tokens_per_task": 9903.5625, "last_run_timestamp": "2026-09-26T08:05:12.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03810672, "avg_duration_per_task_ms": 107015.82, "avg_output_tokens_per_task": 17830.82, "last_run_timestamp": "2026-10-03T08:11:26.752Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8051851111111111, "accuracy_std_dev": 0.020068562864143984, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.87284366, "avg_duration_per_task_ms": 135007.57555555555, "avg_output_tokens_per_task": 5778.728888888889, "last_run_timestamp": "2026-10-03T08:06:43.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.8, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.030123700000000003, "avg_duration_per_task_ms": 294712.06, "avg_output_tokens_per_task": 14851.14, "last_run_timestamp": "2026-07-18T08:03:40.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8, "accuracy_std_dev": 0.006667000000000034, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.053672830000000005, "avg_duration_per_task_ms": 853001.35, "avg_output_tokens_per_task": 37564.6, "last_run_timestamp": "2026-10-03T08:15:57.209Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.7985185555555555, "accuracy_std_dev": 0.0278934726458882, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.9265346288888889, "avg_duration_per_task_ms": 128415.08888888889, "avg_output_tokens_per_task": 5765.7555555555555, "last_run_timestamp": "2026-10-03T08:17:58.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7952381428571429, "accuracy_std_dev": 0.02038185356374333, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.8750597885714285, "avg_duration_per_task_ms": 147569.17142857143, "avg_output_tokens_per_task": 5729.828571428571, "last_run_timestamp": "2026-10-03T08:21:31.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.020424579999999998, "avg_duration_per_task_ms": 413179.4, "avg_output_tokens_per_task": 22096.3, "last_run_timestamp": "2026-10-03T08:38:04.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7925926666666666, "accuracy_std_dev": 0.01616105838118283, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.8220114666666667, "avg_duration_per_task_ms": 137835.87333333332, "avg_output_tokens_per_task": 5751.835555555555, "last_run_timestamp": "2026-10-03T08:17:58.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.79, "accuracy_std_dev": 0.01666640000120001, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.016075525, "avg_duration_per_task_ms": 177598.75, "avg_output_tokens_per_task": 11878.23, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7896079411764706, "accuracy_std_dev": 0.020702777175085597, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.9446278929411763, "avg_duration_per_task_ms": 97552.96117647059, "avg_output_tokens_per_task": 5626.15294117647, "last_run_timestamp": "2026-10-03T08:25:13.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.7871621216216216, "accuracy_std_dev": 0.015496260115123086, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06734989864864864, "avg_duration_per_task_ms": 463154.4662162162, "avg_output_tokens_per_task": 9087.885135135135, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14809964, "avg_duration_per_task_ms": 328566.04, "avg_output_tokens_per_task": 11598.26, "last_run_timestamp": "2026-07-18T08:04:04.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7862320000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.029582717391304348, "avg_duration_per_task_ms": 2435445.304347826, "avg_output_tokens_per_task": 108721.58695652174, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7859532642140468, "accuracy_std_dev": 0.022060806735148097, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028781180602006688, "avg_duration_per_task_ms": 346700.6989966555, "avg_output_tokens_per_task": 11874.802675585284, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7859258888888889, "accuracy_std_dev": 0.023611001599746027, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4867972777777778, "avg_duration_per_task_ms": 127433.78, "avg_output_tokens_per_task": 7132.633333333333, "last_run_timestamp": "2026-10-03T08:09:09.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.785, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08538871499999999, "avg_duration_per_task_ms": 41332.83, "avg_output_tokens_per_task": 4686.75, "last_run_timestamp": "2026-10-03T08:21:29.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7847225, "accuracy_std_dev": 0.020833500000000005, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.11493318749999999, "avg_duration_per_task_ms": 147359.25, "avg_output_tokens_per_task": 14387.041666666666, "last_run_timestamp": "2026-10-01T16:00:58.391Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.784471297188755, "accuracy_std_dev": 0.02769278737577711, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.09832240562248996, "avg_duration_per_task_ms": 406639.9036144578, "avg_output_tokens_per_task": 17335.586345381525, "last_run_timestamp": "2026-09-26T08:23:09.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.784090875, "accuracy_std_dev": 0.025439439704308266, "run_count": 9, "total_questions": 440, "avg_cost_per_task_usd": 0.07952366136363637, "avg_duration_per_task_ms": 1164717.7386363635, "avg_output_tokens_per_task": 26062.15681818182, "last_run_timestamp": "2026-10-03T16:05:34.190Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7835159358372455, "accuracy_std_dev": 0.020461293571987414, "run_count": 13, "total_questions": 639, "avg_cost_per_task_usd": 0.08511225195618151, "avg_duration_per_task_ms": 86820.21752738654, "avg_output_tokens_per_task": 4360.830985915493, "last_run_timestamp": "2026-10-03T08:39:04.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7833639159049359, "accuracy_std_dev": 0.011285272864949418, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.10846238025594149, "avg_duration_per_task_ms": 142469.36563071297, "avg_output_tokens_per_task": 11291.52285191956, "last_run_timestamp": "2026-10-03T08:33:36.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7827005274261605, "accuracy_std_dev": 0.02374291531720566, "run_count": 5, "total_questions": 237, "avg_cost_per_task_usd": 0.10862543037974683, "avg_duration_per_task_ms": 71286.46835443038, "avg_output_tokens_per_task": 8041.662447257384, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.782313, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.01855418367346939, "avg_duration_per_task_ms": 64182.816326530614, "avg_output_tokens_per_task": 11933.040816326531, "last_run_timestamp": "2026-10-08T18:21:28.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.78166675, "accuracy_std_dev": 0.022047820503113248, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06611864500000002, "avg_duration_per_task_ms": 311561.6025, "avg_output_tokens_per_task": 15444.9325, "last_run_timestamp": "2026-10-03T16:18:22.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7811890906432747, "accuracy_std_dev": 0.04113861939038473, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.11509685380116959, "avg_duration_per_task_ms": 3147556.2339181285, "avg_output_tokens_per_task": 25903.42105263158, "last_run_timestamp": "2026-10-05T21:59:19.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7811111666666667, "accuracy_std_dev": 0.023544066905957912, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09874701999999999, "avg_duration_per_task_ms": 77113.90666666666, "avg_output_tokens_per_task": 4731.806666666666, "last_run_timestamp": "2026-10-04T21:00:06.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7807408333333333, "accuracy_std_dev": 0.022758947352268594, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.47270033222222213, "avg_duration_per_task_ms": 109166.16777777778, "avg_output_tokens_per_task": 7230.745555555555, "last_run_timestamp": "2026-10-03T08:13:54.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7800003333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2757919799999997, "avg_duration_per_task_ms": 102078.39333333333, "avg_output_tokens_per_task": 5685.653333333334, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.78, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12438624000000001, "avg_duration_per_task_ms": 511620.62, "avg_output_tokens_per_task": 12374.38, "last_run_timestamp": "2026-10-03T13:23:27.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06720102, "avg_duration_per_task_ms": 196852.88, "avg_output_tokens_per_task": 12773.4, "last_run_timestamp": "2026-10-09T15:19:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01813584, "avg_duration_per_task_ms": 915938.12, "avg_output_tokens_per_task": 36481, "last_run_timestamp": "2026-10-03T08:17:09.266Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12014696000000001, "avg_duration_per_task_ms": 363719.42, "avg_output_tokens_per_task": 9566.28, "last_run_timestamp": "2026-07-18T16:00:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7790475714285714, "accuracy_std_dev": 0.02348347998583042, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4780597828571429, "avg_duration_per_task_ms": 128145.17428571428, "avg_output_tokens_per_task": 7892.805714285714, "last_run_timestamp": "2026-10-03T08:15:57.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.778333375, "accuracy_std_dev": 0.020749956493553795, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1860782475, "avg_duration_per_task_ms": 191849.3575, "avg_output_tokens_per_task": 9582.0075, "last_run_timestamp": "2026-10-03T16:10:17.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.77833325, "accuracy_std_dev": 0.011902275693643617, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.19232103999999997, "avg_duration_per_task_ms": 75116.48, "avg_output_tokens_per_task": 6244.94, "last_run_timestamp": "2026-10-03T08:17:58.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.777857107142857, "accuracy_std_dev": 0.024498453027457068, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.5571781464285714, "avg_duration_per_task_ms": 116452.10642857142, "avg_output_tokens_per_task": 7231.447142857143, "last_run_timestamp": "2026-10-03T08:36:29.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7777776666666667, "accuracy_std_dev": 0.021998866415239553, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08069113666666665, "avg_duration_per_task_ms": 131032.13666666667, "avg_output_tokens_per_task": 4578.13, "last_run_timestamp": "2026-10-03T08:38:05.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7772839629629629, "accuracy_std_dev": 0.02231524453767886, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.4874687607407408, "avg_duration_per_task_ms": 125767.80296296296, "avg_output_tokens_per_task": 7179.893333333333, "last_run_timestamp": "2026-10-03T08:32:03.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7766669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.18507052999999998, "avg_duration_per_task_ms": 70446.4, "avg_output_tokens_per_task": 6214.06, "last_run_timestamp": "2026-10-03T08:13:12.599Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.011385435794128478, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.18964581, "avg_duration_per_task_ms": 65753.47666666667, "avg_output_tokens_per_task": 6306.6466666666665, "last_run_timestamp": "2026-10-03T08:23:24.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.02841481095407032, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21599287333333336, "avg_duration_per_task_ms": 589051.7166666667, "avg_output_tokens_per_task": 12452.323333333334, "last_run_timestamp": "2026-10-08T22:44:43.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.026874233841275463, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15658131666666666, "avg_duration_per_task_ms": 581092.8233333334, "avg_output_tokens_per_task": 12459.696666666667, "last_run_timestamp": "2026-10-03T08:17:09.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7762865704697987, "accuracy_std_dev": 0.02285360036308408, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.07428861073825505, "avg_duration_per_task_ms": 1222334.8724832216, "avg_output_tokens_per_task": 35964.32214765101, "last_run_timestamp": "2026-10-08T23:26:25.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7761905714285714, "accuracy_std_dev": 0.024907983039620864, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.18391273428571428, "avg_duration_per_task_ms": 728509.9057142857, "avg_output_tokens_per_task": 29807.6, "last_run_timestamp": "2026-10-03T16:06:04.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7760001, "accuracy_std_dev": 0.021333425001391598, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.44147494200000004, "avg_duration_per_task_ms": 131608.992, "avg_output_tokens_per_task": 7283.092, "last_run_timestamp": "2026-10-03T08:36:29.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.776, "accuracy_std_dev": 0.012364752775530953, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.074485596, "avg_duration_per_task_ms": 47627.076, "avg_output_tokens_per_task": 4707.808, "last_run_timestamp": "2026-10-03T08:37:28.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7755376129032258, "accuracy_std_dev": 0.01731650665001458, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.1579919798387097, "avg_duration_per_task_ms": 333040.58467741933, "avg_output_tokens_per_task": 20363.45564516129, "last_run_timestamp": "2026-10-03T08:32:03.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.775, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029526945, "avg_duration_per_task_ms": 114391.885, "avg_output_tokens_per_task": 11970.57, "last_run_timestamp": "2026-10-03T16:15:04.036Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7744441666666667, "accuracy_std_dev": 0.015112691911289488, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09511451000000001, "avg_duration_per_task_ms": 133903.96, "avg_output_tokens_per_task": 4573.283333333334, "last_run_timestamp": "2026-10-03T08:23:53.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.7744105858585859, "accuracy_std_dev": 0.019115499999999952, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.24473854545454546, "avg_duration_per_task_ms": 51240.79797979798, "avg_output_tokens_per_task": 4631.454545454545, "last_run_timestamp": "2026-10-03T08:37:57.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.017612503291811795, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12337239714285714, "avg_duration_per_task_ms": 82299.41428571429, "avg_output_tokens_per_task": 4508.834285714286, "last_run_timestamp": "2026-10-03T08:19:27.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7741665000000001, "accuracy_std_dev": 0.04514642535406319, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12616785249999998, "avg_duration_per_task_ms": 205632.94, "avg_output_tokens_per_task": 7251.665, "last_run_timestamp": "2026-10-05T19:59:14.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.77375, "accuracy_std_dev": 0.02188582748035816, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.127833135, "avg_duration_per_task_ms": 303219.4325, "avg_output_tokens_per_task": 19468.2, "last_run_timestamp": "2026-10-09T22:14:36.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.02914423052490644, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4720031333333333, "avg_duration_per_task_ms": 115130.24444444444, "avg_output_tokens_per_task": 7206.9, "last_run_timestamp": "2026-10-03T08:16:37.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.77333325, "accuracy_std_dev": 0.017638436566794497, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4344371716666668, "avg_duration_per_task_ms": 95528.38, "avg_output_tokens_per_task": 5662.913333333333, "last_run_timestamp": "2026-10-03T08:06:28.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.77333325, "accuracy_std_dev": 0.004713927363409425, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007096445, "avg_duration_per_task_ms": 164831.185, "avg_output_tokens_per_task": 17141.055, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11157742000000001, "avg_duration_per_task_ms": 52237.66, "avg_output_tokens_per_task": 12874.88, "last_run_timestamp": "2026-07-18T08:03:41.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.773333, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06652479, "avg_duration_per_task_ms": 59089.94, "avg_output_tokens_per_task": 12443.51, "last_run_timestamp": "2026-10-05T17:41:05.783Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02012318, "avg_duration_per_task_ms": 229027.46, "avg_output_tokens_per_task": 10940.04, "last_run_timestamp": "2026-07-18T08:04:46.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.042562220000000005, "avg_duration_per_task_ms": 321425.02, "avg_output_tokens_per_task": 8985.7, "last_run_timestamp": "2026-07-18T16:00:10.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7729411176470589, "accuracy_std_dev": 0.02923599978246472, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.5054271811764706, "avg_duration_per_task_ms": 128787.48470588235, "avg_output_tokens_per_task": 7945.302352941177, "last_run_timestamp": "2026-10-03T08:15:30.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.77277775, "accuracy_std_dev": 0.019333070721456368, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4942197533333333, "avg_duration_per_task_ms": 128626.26333333334, "avg_output_tokens_per_task": 8065.555, "last_run_timestamp": "2026-10-03T08:20:22.745Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7727776666666667, "accuracy_std_dev": 0.03324520758278134, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11460699333333334, "avg_duration_per_task_ms": 172737.43666666668, "avg_output_tokens_per_task": 9388.596666666666, "last_run_timestamp": "2026-10-03T16:16:11.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7725, "accuracy_std_dev": 0.021457321984581383, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17566200249999997, "avg_duration_per_task_ms": 370370.8425, "avg_output_tokens_per_task": 27669.585, "last_run_timestamp": "2026-10-03T16:19:10.407Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.772381, "accuracy_std_dev": 0.020909200094011936, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.17040614571428572, "avg_duration_per_task_ms": 747500.4285714285, "avg_output_tokens_per_task": 28014.845714285715, "last_run_timestamp": "2026-10-03T16:14:18.634Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7722222222222223, "accuracy_std_dev": 0.03095202181433487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03593436666666667, "avg_duration_per_task_ms": 446735.1666666667, "avg_output_tokens_per_task": 13815.786666666667, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7722221666666667, "accuracy_std_dev": 0.0225805089137857, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.009277203333333336, "avg_duration_per_task_ms": 211225.32333333333, "avg_output_tokens_per_task": 4258.733333333334, "last_run_timestamp": "2026-10-03T08:53:37.283Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.772109, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.06601828571428571, "avg_duration_per_task_ms": 84701.85714285714, "avg_output_tokens_per_task": 5801.673469387755, "last_run_timestamp": "2026-07-18T08:06:26.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.7718518888888889, "accuracy_std_dev": 0.027176331186630626, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04377958222222222, "avg_duration_per_task_ms": 219884.14888888889, "avg_output_tokens_per_task": 9569.68888888889, "last_run_timestamp": "2026-10-03T16:15:21.561Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7718518333333333, "accuracy_std_dev": 0.03201514259473338, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.43984256000000005, "avg_duration_per_task_ms": 80156.35111111111, "avg_output_tokens_per_task": 5674.794444444445, "last_run_timestamp": "2026-10-03T08:11:53.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.77166675, "accuracy_std_dev": 0.027233420032884263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08857231500000001, "avg_duration_per_task_ms": 275121.275, "avg_output_tokens_per_task": 19864.29, "last_run_timestamp": "2026-10-03T08:38:07.639Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.77166675, "accuracy_std_dev": 0.02020715632362706, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09734501, "avg_duration_per_task_ms": 129281.66, "avg_output_tokens_per_task": 5411.39, "last_run_timestamp": "2026-10-03T08:23:24.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.77166675, "accuracy_std_dev": 0.009860203404975986, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.11018350999999998, "avg_duration_per_task_ms": 71664.89, "avg_output_tokens_per_task": 4808.815, "last_run_timestamp": "2026-10-03T08:21:31.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.771258387755102, "accuracy_std_dev": 0.022567909478948184, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.06584808673469388, "avg_duration_per_task_ms": 190152.3418367347, "avg_output_tokens_per_task": 10271.474489795919, "last_run_timestamp": "2026-10-08T23:02:20.692Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7711112222222222, "accuracy_std_dev": 0.021773400922418973, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5318819755555556, "avg_duration_per_task_ms": 174829.24, "avg_output_tokens_per_task": 10576.931111111111, "last_run_timestamp": "2026-10-03T08:10:31.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7711111666666667, "accuracy_std_dev": 0.016629684897963515, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06979356333333334, "avg_duration_per_task_ms": 35038.676666666666, "avg_output_tokens_per_task": 4742.786666666667, "last_run_timestamp": "2026-10-03T08:30:16.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.771111, "accuracy_std_dev": 0.04085521560339636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4149127733333333, "avg_duration_per_task_ms": 472514.2266666667, "avg_output_tokens_per_task": 36470.113333333335, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.77083325, "accuracy_std_dev": 0.028662434281433592, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0230241875, "avg_duration_per_task_ms": 301656.3375, "avg_output_tokens_per_task": 11912.9475, "last_run_timestamp": "2026-10-03T16:14:24.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7706668, "accuracy_std_dev": 0.010832125062055002, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.100171232, "avg_duration_per_task_ms": 273464.56, "avg_output_tokens_per_task": 22455.776, "last_run_timestamp": "2026-10-03T08:37:40.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.770580068825911, "accuracy_std_dev": 0.030947081374501206, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.08480364777327934, "avg_duration_per_task_ms": 268608.14574898785, "avg_output_tokens_per_task": 10039.971659919029, "last_run_timestamp": "2026-10-03T16:17:43.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7704762857142857, "accuracy_std_dev": 0.03627188525453795, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07425714857142857, "avg_duration_per_task_ms": 258403.66, "avg_output_tokens_per_task": 11728.06, "last_run_timestamp": "2026-10-03T08:42:46.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.770476, "accuracy_std_dev": 0.02332836876300735, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07879788857142857, "avg_duration_per_task_ms": 228548.80857142858, "avg_output_tokens_per_task": 9318.671428571428, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7700001666666666, "accuracy_std_dev": 0.02168818589629438, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08111510333333333, "avg_duration_per_task_ms": 263647.49333333335, "avg_output_tokens_per_task": 13148.976666666667, "last_run_timestamp": "2026-10-03T08:16:12.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.18774403, "avg_duration_per_task_ms": 78191.08, "avg_output_tokens_per_task": 6339.04, "last_run_timestamp": "2026-10-03T08:11:23.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.77, "accuracy_std_dev": 0.018257509872652396, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.049748467500000004, "avg_duration_per_task_ms": 175035.1425, "avg_output_tokens_per_task": 11732.075, "last_run_timestamp": "2026-10-03T16:16:48.795Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.77, "accuracy_std_dev": 0.03986093883114146, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1172345275, "avg_duration_per_task_ms": 283399.5725, "avg_output_tokens_per_task": 12335.785, "last_run_timestamp": "2026-10-03T16:16:31.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.77, "accuracy_std_dev": 0.017950332712793934, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.10653246, "avg_duration_per_task_ms": 360324.605, "avg_output_tokens_per_task": 20634.68, "last_run_timestamp": "2026-10-03T08:06:28.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7696969898989898, "accuracy_std_dev": 0.023999951853285028, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.05881589494949495, "avg_duration_per_task_ms": 254508.298989899, "avg_output_tokens_per_task": 11128.349494949494, "last_run_timestamp": "2026-10-09T15:53:11.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7696295555555556, "accuracy_std_dev": 0.019654008621998214, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4308894377777778, "avg_duration_per_task_ms": 100253.29111111112, "avg_output_tokens_per_task": 5601.184444444444, "last_run_timestamp": "2026-10-03T08:11:39.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7696295555555556, "accuracy_std_dev": 0.027952376707182647, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08778874, "avg_duration_per_task_ms": 191583.55333333334, "avg_output_tokens_per_task": 11553.07111111111, "last_run_timestamp": "2026-10-10T01:08:05.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7695237142857143, "accuracy_std_dev": 0.01326512827695538, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10013912285714285, "avg_duration_per_task_ms": 333739.5742857143, "avg_output_tokens_per_task": 20222.222857142857, "last_run_timestamp": "2026-10-03T08:17:48.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7694445, "accuracy_std_dev": 0.02899006747864978, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.551655145, "avg_duration_per_task_ms": 183576.22666666665, "avg_output_tokens_per_task": 10538.358333333334, "last_run_timestamp": "2026-10-03T08:10:31.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.7693334, "accuracy_std_dev": 0.027194796889110983, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.047570167999999996, "avg_duration_per_task_ms": 114696.804, "avg_output_tokens_per_task": 4956.572, "last_run_timestamp": "2026-10-08T23:21:59.850Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.021998731726069008, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09783254, "avg_duration_per_task_ms": 102292.53333333334, "avg_output_tokens_per_task": 23933.02, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0562409, "avg_duration_per_task_ms": 124558.21333333333, "avg_output_tokens_per_task": 4508.253333333333, "last_run_timestamp": "2026-09-26T08:24:47.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.768707, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.18590030612244896, "avg_duration_per_task_ms": 362695.9183673469, "avg_output_tokens_per_task": 6540.102040816327, "last_run_timestamp": "2026-09-29T21:00:05.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7683335, "accuracy_std_dev": 0.017873086533948198, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14871092, "avg_duration_per_task_ms": 286909.625, "avg_output_tokens_per_task": 13559.755, "last_run_timestamp": "2026-10-03T16:12:29.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7683335, "accuracy_std_dev": 0.01190259075789804, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15200416500000002, "avg_duration_per_task_ms": 286481.905, "avg_output_tokens_per_task": 19800.085, "last_run_timestamp": "2026-10-03T08:10:31.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.768333375, "accuracy_std_dev": 0.028625995860133417, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0977644525, "avg_duration_per_task_ms": 232629.3425, "avg_output_tokens_per_task": 11722.16, "last_run_timestamp": "2026-10-03T16:04:46.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.76833325, "accuracy_std_dev": 0.023746444179866155, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.038813765, "avg_duration_per_task_ms": 338234.72, "avg_output_tokens_per_task": 12670.62, "last_run_timestamp": "2026-10-09T00:07:53.060Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.767778, "accuracy_std_dev": 0.009557997105391226, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.055267166666666666, "avg_duration_per_task_ms": 921928.7333333333, "avg_output_tokens_per_task": 38360.306666666664, "last_run_timestamp": "2026-10-03T08:15:16.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7677777777777778, "accuracy_std_dev": 0.03175926811456523, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.5635850022222223, "avg_duration_per_task_ms": 180702.17222222223, "avg_output_tokens_per_task": 10553.803333333333, "last_run_timestamp": "2026-10-03T08:12:06.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7673375906040268, "accuracy_std_dev": 0.01364674745945795, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.02802279865771812, "avg_duration_per_task_ms": 90972.77852348993, "avg_output_tokens_per_task": 17409.81879194631, "last_run_timestamp": "2026-10-03T08:20:48.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0121027, "avg_duration_per_task_ms": 95348.66, "avg_output_tokens_per_task": 10736.12, "last_run_timestamp": "2026-10-08T06:08:26.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025223939999999997, "avg_duration_per_task_ms": 165137.1, "avg_output_tokens_per_task": 14240.82, "last_run_timestamp": "2026-09-15T00:00:05.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0668136, "avg_duration_per_task_ms": 100184.26, "avg_output_tokens_per_task": 11544.38, "last_run_timestamp": "2026-07-18T08:04:03.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01215033, "avg_duration_per_task_ms": 551566.9, "avg_output_tokens_per_task": 23566.32, "last_run_timestamp": "2026-10-03T08:18:05.266Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.014401440259764175, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3460653, "avg_duration_per_task_ms": 170899.55333333334, "avg_output_tokens_per_task": 12611.153333333334, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.020727473271535585, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06962397, "avg_duration_per_task_ms": 57268.18666666667, "avg_output_tokens_per_task": 4623.046666666667, "last_run_timestamp": "2026-10-03T08:43:42.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7666666153846154, "accuracy_std_dev": 0.020085283979679725, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.5291991723076925, "avg_duration_per_task_ms": 121619.19230769231, "avg_output_tokens_per_task": 8086.011538461538, "last_run_timestamp": "2026-10-03T08:16:24.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.7666666, "accuracy_std_dev": 0.009428137559454682, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.081377272, "avg_duration_per_task_ms": 89133.128, "avg_output_tokens_per_task": 4543.572, "last_run_timestamp": "2026-10-03T08:30:16.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7666666, "accuracy_std_dev": 0.018856133692780202, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.061894704, "avg_duration_per_task_ms": 48771.608, "avg_output_tokens_per_task": 4659.024, "last_run_timestamp": "2026-10-03T08:19:53.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7666665714285714, "accuracy_std_dev": 0.01951809905012813, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08535362285714285, "avg_duration_per_task_ms": 334395.88857142854, "avg_output_tokens_per_task": 19919.711428571427, "last_run_timestamp": "2026-10-03T08:40:08.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7661904285714286, "accuracy_std_dev": 0.01872337959311179, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019808859999999998, "avg_duration_per_task_ms": 510174.85142857145, "avg_output_tokens_per_task": 13353.934285714286, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.765833375, "accuracy_std_dev": 0.03471108632676851, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04402259, "avg_duration_per_task_ms": 213134.3675, "avg_output_tokens_per_task": 9446.67, "last_run_timestamp": "2026-10-03T16:14:17.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.7653059591836735, "accuracy_std_dev": 0.005416499999999991, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.02433298979591837, "avg_duration_per_task_ms": 682809.275510204, "avg_output_tokens_per_task": 13469.224489795919, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.7651006510067114, "accuracy_std_dev": 0.02769017312403245, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.035872944630872476, "avg_duration_per_task_ms": 526054.0553691275, "avg_output_tokens_per_task": 12761.159395973154, "last_run_timestamp": "2026-10-08T22:51:57.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.765, "accuracy_std_dev": 0.036930670783239215, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.032793829999999996, "avg_duration_per_task_ms": 164299.645, "avg_output_tokens_per_task": 15398.3, "last_run_timestamp": "2026-10-03T08:19:00.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.765, "accuracy_std_dev": 0.028819341500110663, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.011177185, "avg_duration_per_task_ms": 267604.5325, "avg_output_tokens_per_task": 6349.9575, "last_run_timestamp": "2026-10-03T08:39:56.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7647619999999999, "accuracy_std_dev": 0.022172301981139845, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.011946834285714285, "avg_duration_per_task_ms": 275656.4971428571, "avg_output_tokens_per_task": 13715.774285714286, "last_run_timestamp": "2026-10-03T16:20:06.645Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7646564924623115, "accuracy_std_dev": 0.022445600325773765, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.10723165075376884, "avg_duration_per_task_ms": 358580.18341708544, "avg_output_tokens_per_task": 11741.600502512563, "last_run_timestamp": "2026-10-03T16:15:36.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7646153076923077, "accuracy_std_dev": 0.017615963863166925, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.019299193846153844, "avg_duration_per_task_ms": 239187.3676923077, "avg_output_tokens_per_task": 17117.927692307694, "last_run_timestamp": "2026-10-03T08:37:24.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7644446666666667, "accuracy_std_dev": 0.020122943113658996, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4976620088888889, "avg_duration_per_task_ms": 132296.16222222222, "avg_output_tokens_per_task": 8163.442222222222, "last_run_timestamp": "2026-10-03T08:17:09.269Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7644445, "accuracy_std_dev": 0.021999102116147078, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06149954333333334, "avg_duration_per_task_ms": 120941.48333333334, "avg_output_tokens_per_task": 4754.513333333333, "last_run_timestamp": "2026-10-03T08:41:33.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7644445, "accuracy_std_dev": 0.03281078940303024, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08256609000000001, "avg_duration_per_task_ms": 48194.92, "avg_output_tokens_per_task": 4847.44, "last_run_timestamp": "2026-10-03T08:29:04.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029735113333333334, "avg_duration_per_task_ms": 54092.806666666664, "avg_output_tokens_per_task": 4732.093333333333, "last_run_timestamp": "2026-10-03T08:37:59.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7643097626262627, "accuracy_std_dev": 0.041231633865880205, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.02870519191919192, "avg_duration_per_task_ms": 478260.4166666667, "avg_output_tokens_per_task": 12840.517676767677, "last_run_timestamp": "2026-10-03T16:10:32.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7641723061224489, "accuracy_std_dev": 0.03540326230730724, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.01049295918367347, "avg_duration_per_task_ms": 1253925.7823129252, "avg_output_tokens_per_task": 17278.809523809523, "last_run_timestamp": "2026-10-08T23:09:38.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7641127822580646, "accuracy_std_dev": 0.03914012491663765, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.10629601008064517, "avg_duration_per_task_ms": 864547.8709677419, "avg_output_tokens_per_task": 10411.459677419354, "last_run_timestamp": "2026-10-05T20:07:40.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7641026153846153, "accuracy_std_dev": 0.016021001176525294, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.017410869230769233, "avg_duration_per_task_ms": 197270.92, "avg_output_tokens_per_task": 19327.204615384617, "last_run_timestamp": "2026-10-03T08:29:06.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.764, "accuracy_std_dev": 0.02585427538338681, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.050899012, "avg_duration_per_task_ms": 154368.784, "avg_output_tokens_per_task": 4603.632, "last_run_timestamp": "2026-10-03T08:39:30.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01259681, "avg_duration_per_task_ms": 657803.07, "avg_output_tokens_per_task": 25182.87, "last_run_timestamp": "2026-10-03T08:05:49.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02542324, "avg_duration_per_task_ms": 138965.85, "avg_output_tokens_per_task": 13863.01, "last_run_timestamp": "2026-10-03T16:12:10.916Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06322074000000001, "avg_duration_per_task_ms": 968065, "avg_output_tokens_per_task": 43847.04, "last_run_timestamp": "2026-10-03T08:20:05.772Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7631579766081871, "accuracy_std_dev": 0.021601269198350176, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.025081707602339184, "avg_duration_per_task_ms": 429534.44444444444, "avg_output_tokens_per_task": 12901.88596491228, "last_run_timestamp": "2026-10-03T16:09:19.685Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7627686612903226, "accuracy_std_dev": 0.02051209766552413, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.02082460887096774, "avg_duration_per_task_ms": 113248.77822580645, "avg_output_tokens_per_task": 14257.657258064517, "last_run_timestamp": "2026-10-03T08:37:24.028Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7626666, "accuracy_std_dev": 0.03227647784749755, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.146569104, "avg_duration_per_task_ms": 373419.58, "avg_output_tokens_per_task": 24831.152, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.762500125, "accuracy_std_dev": 0.027474623393403872, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12900198, "avg_duration_per_task_ms": 265844.8625, "avg_output_tokens_per_task": 11594.9925, "last_run_timestamp": "2026-10-03T16:12:29.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7625, "accuracy_std_dev": 0.03072062282490381, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.160654465, "avg_duration_per_task_ms": 223937.135, "avg_output_tokens_per_task": 9632.975, "last_run_timestamp": "2026-10-03T16:12:07.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.762381, "accuracy_std_dev": 0.013179314517184024, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12107822285714286, "avg_duration_per_task_ms": 467340.5485714286, "avg_output_tokens_per_task": 21773.494285714285, "last_run_timestamp": "2026-10-03T08:54:39.113Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.011330936570096763, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03153908666666666, "avg_duration_per_task_ms": 312202.32, "avg_output_tokens_per_task": 39820.306666666664, "last_run_timestamp": "2026-10-03T08:09:38.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7621211818181818, "accuracy_std_dev": 0.026372448546363295, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05366102181818182, "avg_duration_per_task_ms": 93039.47818181818, "avg_output_tokens_per_task": 4281.278181818182, "last_run_timestamp": "2026-10-03T08:43:08.437Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7620649764309765, "accuracy_std_dev": 0.008722071727647192, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.007020558922558921, "avg_duration_per_task_ms": 240106.04377104377, "avg_output_tokens_per_task": 12275.050505050505, "last_run_timestamp": "2026-08-29T16:33:29.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.7619048571428572, "accuracy_std_dev": 0.017353196366158268, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01491982857142857, "avg_duration_per_task_ms": 401525.7, "avg_output_tokens_per_task": 15852.922857142858, "last_run_timestamp": "2026-10-03T16:17:41.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.020067932249624297, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.026417691428571432, "avg_duration_per_task_ms": 160543.33428571429, "avg_output_tokens_per_task": 8753.874285714286, "last_run_timestamp": "2026-10-03T16:17:22.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.03611523004698906, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07077762857142857, "avg_duration_per_task_ms": 309481.20857142855, "avg_output_tokens_per_task": 12537.965714285714, "last_run_timestamp": "2026-10-03T08:21:29.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.761666625, "accuracy_std_dev": 0.015184060540065538, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05742875749999999, "avg_duration_per_task_ms": 229440.2175, "avg_output_tokens_per_task": 10184.775, "last_run_timestamp": "2026-10-03T16:05:34.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7613333999999999, "accuracy_std_dev": 0.012219939993306055, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.059212192000000004, "avg_duration_per_task_ms": 137664.628, "avg_output_tokens_per_task": 14571.664, "last_run_timestamp": "2026-10-09T02:14:11.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7613333999999999, "accuracy_std_dev": 0.019955635240202208, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.054660096000000005, "avg_duration_per_task_ms": 124144.4, "avg_output_tokens_per_task": 4448.496, "last_run_timestamp": "2026-10-03T08:45:16.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7613333999999999, "accuracy_std_dev": 0.022469788112930676, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06195499599999999, "avg_duration_per_task_ms": 143749.252, "avg_output_tokens_per_task": 6410.492, "last_run_timestamp": "2026-10-03T08:22:27.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7613332, "accuracy_std_dev": 0.026297374605842312, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.167040694, "avg_duration_per_task_ms": 594740.238, "avg_output_tokens_per_task": 37349.044, "last_run_timestamp": "2026-10-03T08:17:06.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7611113333333333, "accuracy_std_dev": 0.025288354320165317, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17373087333333334, "avg_duration_per_task_ms": 477933.7, "avg_output_tokens_per_task": 24266.54, "last_run_timestamp": "2026-08-29T17:19:03.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7610256923076923, "accuracy_std_dev": 0.021218684472687738, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.589651923076923, "avg_duration_per_task_ms": 120803.91923076923, "avg_output_tokens_per_task": 8142.596923076923, "last_run_timestamp": "2026-10-03T08:38:25.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.0335097115932266, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.43242590285714283, "avg_duration_per_task_ms": 94424.92, "avg_output_tokens_per_task": 5553.762857142857, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7604938148148149, "accuracy_std_dev": 0.020602075859156715, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.656406777037037, "avg_duration_per_task_ms": 73567.76444444444, "avg_output_tokens_per_task": 5646.465185185185, "last_run_timestamp": "2026-10-03T08:42:35.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7600001249999999, "accuracy_std_dev": 0.03349974295213884, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0116075025, "avg_duration_per_task_ms": 155751.2425, "avg_output_tokens_per_task": 14262.745, "last_run_timestamp": "2026-10-03T16:12:33.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.76, "accuracy_std_dev": 0.02735228340559523, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5190067183333332, "avg_duration_per_task_ms": 221047.60166666665, "avg_output_tokens_per_task": 10200.993333333334, "last_run_timestamp": "2026-10-03T08:20:09.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.76, "accuracy_std_dev": 0.020870148806369355, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.0067541, "avg_duration_per_task_ms": 260092.14, "avg_output_tokens_per_task": 15338.358, "last_run_timestamp": "2026-10-03T08:38:03.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.01817028797633262, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08873829714285716, "avg_duration_per_task_ms": 247182.05714285714, "avg_output_tokens_per_task": 16018.762857142858, "last_run_timestamp": "2026-10-03T16:22:04.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.76, "accuracy_std_dev": 0.013984245106547586, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.16696927200000003, "avg_duration_per_task_ms": 223419.404, "avg_output_tokens_per_task": 11293.668, "last_run_timestamp": "2026-10-03T16:10:46.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.76, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12531742999999998, "avg_duration_per_task_ms": 312784.77, "avg_output_tokens_per_task": 20756.6, "last_run_timestamp": "2026-10-03T08:29:42.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015769889999999998, "avg_duration_per_task_ms": 1152795.83, "avg_output_tokens_per_task": 36264.12, "last_run_timestamp": "2026-10-03T08:16:24.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.76, "accuracy_std_dev": 0.019626323972325197, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02215426, "avg_duration_per_task_ms": 122620.84, "avg_output_tokens_per_task": 8751.653333333334, "last_run_timestamp": "2026-10-03T16:07:50.330Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.020470931774452417, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09495435142857143, "avg_duration_per_task_ms": 338628.4542857143, "avg_output_tokens_per_task": 12897.748571428572, "last_run_timestamp": "2026-10-03T16:13:59.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.30272262, "avg_duration_per_task_ms": 355584.56, "avg_output_tokens_per_task": 10613.82, "last_run_timestamp": "2026-10-03T18:42:35.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07523408, "avg_duration_per_task_ms": 676626.08, "avg_output_tokens_per_task": 16285.62, "last_run_timestamp": "2026-10-03T08:15:42.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.76, "accuracy_std_dev": 1.1102230246251565e-16, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05899852666666667, "avg_duration_per_task_ms": 72162.04666666666, "avg_output_tokens_per_task": 4660.633333333333, "last_run_timestamp": "2026-10-03T08:13:12.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.020547857418718854, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011870755, "avg_duration_per_task_ms": 198678.775, "avg_output_tokens_per_task": 4474.35, "last_run_timestamp": "2026-10-03T08:39:24.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7599998333333334, "accuracy_std_dev": 0.01333333333437501, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03796188999999999, "avg_duration_per_task_ms": 83545.53666666667, "avg_output_tokens_per_task": 4558.626666666667, "last_run_timestamp": "2026-10-03T08:35:08.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7597841862348177, "accuracy_std_dev": 0.0190545430278451, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.164671975708502, "avg_duration_per_task_ms": 858653.5303643724, "avg_output_tokens_per_task": 31997.372469635626, "last_run_timestamp": "2026-10-09T00:08:43.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7595079060402684, "accuracy_std_dev": 0.0216087995861768, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.1067814966442953, "avg_duration_per_task_ms": 41164.906040268455, "avg_output_tokens_per_task": 4181.802013422819, "last_run_timestamp": "2026-10-03T21:42:23.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7594443333333334, "accuracy_std_dev": 0.015265178644731146, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.008844093333333332, "avg_duration_per_task_ms": 361189.67333333334, "avg_output_tokens_per_task": 14258.985, "last_run_timestamp": "2026-10-03T08:39:28.580Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7593333, "accuracy_std_dev": 0.016452043441773433, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.083344808, "avg_duration_per_task_ms": 106115.21, "avg_output_tokens_per_task": 23741.31, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7592593333333333, "accuracy_std_dev": 0.02381917591633544, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.050810793333333326, "avg_duration_per_task_ms": 122662.1288888889, "avg_output_tokens_per_task": 4548.155555555555, "last_run_timestamp": "2026-10-04T12:00:02.814Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7592592222222222, "accuracy_std_dev": 0.020714267497322178, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.41923416888888887, "avg_duration_per_task_ms": 97992.62444444445, "avg_output_tokens_per_task": 5633.653333333334, "last_run_timestamp": "2026-10-03T08:15:30.343Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7590477142857143, "accuracy_std_dev": 0.013997181448055748, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.24529245999999996, "avg_duration_per_task_ms": 151053.55142857143, "avg_output_tokens_per_task": 23224.54, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7590477142857143, "accuracy_std_dev": 0.034992667843071215, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08880086285714285, "avg_duration_per_task_ms": 68112.38857142857, "avg_output_tokens_per_task": 12261.222857142857, "last_run_timestamp": "2026-10-03T16:15:11.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7588384343434343, "accuracy_std_dev": 0.024611570108132058, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20684339646464647, "avg_duration_per_task_ms": 493166.0580808081, "avg_output_tokens_per_task": 20296.997474747473, "last_run_timestamp": "2026-10-03T08:19:53.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7586668, "accuracy_std_dev": 0.01654634971708262, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.071547176, "avg_duration_per_task_ms": 127553.92, "avg_output_tokens_per_task": 4587.528, "last_run_timestamp": "2026-10-03T09:01:39.121Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7585565223214286, "accuracy_std_dev": 0.0371971128592224, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.19265237276785713, "avg_duration_per_task_ms": 474084.81696428574, "avg_output_tokens_per_task": 27864.78348214286, "last_run_timestamp": "2026-10-03T16:15:58.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.758333375, "accuracy_std_dev": 0.019364862941275235, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0383828675, "avg_duration_per_task_ms": 246490.5475, "avg_output_tokens_per_task": 12250.2775, "last_run_timestamp": "2026-10-03T16:07:44.182Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.025110926131616508, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021590805, "avg_duration_per_task_ms": 448620.805, "avg_output_tokens_per_task": 13949.42, "last_run_timestamp": "2026-10-05T20:08:33.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.40697005500000005, "avg_duration_per_task_ms": 402356.405, "avg_output_tokens_per_task": 35595.7, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7583331666666667, "accuracy_std_dev": 0.013844473427047428, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09591996333333334, "avg_duration_per_task_ms": 214673.9, "avg_output_tokens_per_task": 9689.54, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7583329999999999, "accuracy_std_dev": 0.04499999999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06691538, "avg_duration_per_task_ms": 350031.58, "avg_output_tokens_per_task": 54823.52, "last_run_timestamp": "2026-10-09T03:50:20.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7580951428571429, "accuracy_std_dev": 0.025125616384357164, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5482415914285714, "avg_duration_per_task_ms": 178558.36285714287, "avg_output_tokens_per_task": 10788.502857142857, "last_run_timestamp": "2026-10-03T08:08:31.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.7580951428571429, "accuracy_std_dev": 0.05160461492216857, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02014420857142857, "avg_duration_per_task_ms": 210702.71142857143, "avg_output_tokens_per_task": 12469.117142857143, "last_run_timestamp": "2026-10-03T16:19:59.478Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.758, "accuracy_std_dev": 0.03709447398198281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.079319272, "avg_duration_per_task_ms": 620750.224, "avg_output_tokens_per_task": 10895.356, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7579964141414142, "accuracy_std_dev": 0.023533015514113357, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20713992929292932, "avg_duration_per_task_ms": 664979.7727272727, "avg_output_tokens_per_task": 53099.67929292929, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7577778333333334, "accuracy_std_dev": 0.024545289459260582, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.026035279999999997, "avg_duration_per_task_ms": 169995.66166666665, "avg_output_tokens_per_task": 18134.881666666668, "last_run_timestamp": "2026-10-03T08:33:35.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.04629468145358481, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12420428666666666, "avg_duration_per_task_ms": 116279.86, "avg_output_tokens_per_task": 19029.046666666665, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.35916456, "avg_duration_per_task_ms": 165675.65333333332, "avg_output_tokens_per_task": 12758.866666666667, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.757619, "accuracy_std_dev": 0.014553057204587645, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3625878914285714, "avg_duration_per_task_ms": 337462.2485714286, "avg_output_tokens_per_task": 10492.708571428571, "last_run_timestamp": "2026-10-03T08:37:29.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.757499875, "accuracy_std_dev": 0.03298787550918331, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08512432500000001, "avg_duration_per_task_ms": 226499.755, "avg_output_tokens_per_task": 9952.425, "last_run_timestamp": "2026-10-03T16:09:05.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7572707181208054, "accuracy_std_dev": 0.01805706983120156, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.00953123489932886, "avg_duration_per_task_ms": 371076.8926174497, "avg_output_tokens_per_task": 15024.973154362417, "last_run_timestamp": "2026-10-05T08:35:06.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7571428571428571, "accuracy_std_dev": 0.03263217671225126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01064094, "avg_duration_per_task_ms": 1272417.7228571428, "avg_output_tokens_per_task": 12476.905714285715, "last_run_timestamp": "2026-10-05T20:14:35.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.757037, "accuracy_std_dev": 0.028822348820933333, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5493601955555556, "avg_duration_per_task_ms": 192839.5377777778, "avg_output_tokens_per_task": 10676.25111111111, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17767661, "avg_duration_per_task_ms": 84052.21, "avg_output_tokens_per_task": 6226.55, "last_run_timestamp": "2026-10-03T08:17:58.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02875953, "avg_duration_per_task_ms": 155469.3, "avg_output_tokens_per_task": 15776.48, "last_run_timestamp": "2026-10-03T08:38:11.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.756525067114094, "accuracy_std_dev": 0.028844409691384666, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.0700060134228188, "avg_duration_per_task_ms": 265527.27293064876, "avg_output_tokens_per_task": 11642.608501118568, "last_run_timestamp": "2026-10-03T13:30:10.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.7564103277591973, "accuracy_std_dev": 0.017294806587206737, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.0331466856187291, "avg_duration_per_task_ms": 189272.61872909698, "avg_output_tokens_per_task": 17663.859531772574, "last_run_timestamp": "2026-10-09T21:37:39.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7563351081871345, "accuracy_std_dev": 0.013338594901982123, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.033182684210526316, "avg_duration_per_task_ms": 2146298.769005848, "avg_output_tokens_per_task": 50213.23391812865, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7561905714285714, "accuracy_std_dev": 0.0243929472000011, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1949313942857143, "avg_duration_per_task_ms": 124043.37428571428, "avg_output_tokens_per_task": 7684.874285714286, "last_run_timestamp": "2026-10-03T08:17:09.273Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7561538461538462, "accuracy_std_dev": 0.031169392555582855, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.2008702830769231, "avg_duration_per_task_ms": 120980.52153846154, "avg_output_tokens_per_task": 7797.5623076923075, "last_run_timestamp": "2026-10-03T08:16:37.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.756, "accuracy_std_dev": 0.02048314212224288, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022952103999999997, "avg_duration_per_task_ms": 125535.02, "avg_output_tokens_per_task": 15581.832, "last_run_timestamp": "2026-10-03T08:16:37.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.756, "accuracy_std_dev": 0.021745967699782896, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.006672712000000001, "avg_duration_per_task_ms": 144287.16, "avg_output_tokens_per_task": 4265.224, "last_run_timestamp": "2026-10-03T08:12:06.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7557079178082191, "accuracy_std_dev": 0.01607886717264481, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.2608585205479452, "avg_duration_per_task_ms": 125025.47260273973, "avg_output_tokens_per_task": 7152.054794520548, "last_run_timestamp": "2026-10-03T08:05:09.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7556891325301206, "accuracy_std_dev": 0.028686868714448425, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.05997379919678716, "avg_duration_per_task_ms": 208407.72289156626, "avg_output_tokens_per_task": 7587.301204819277, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7556161712204007, "accuracy_std_dev": 0.028091014564674013, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.08449569216757742, "avg_duration_per_task_ms": 138097.57559198543, "avg_output_tokens_per_task": 12733.64116575592, "last_run_timestamp": "2026-10-05T17:44:38.668Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7555803571428571, "accuracy_std_dev": 0.023882484930784247, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.044577171874999995, "avg_duration_per_task_ms": 507800.79910714284, "avg_output_tokens_per_task": 22983.540178571428, "last_run_timestamp": "2026-10-09T00:21:32.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.027460826666666667, "avg_duration_per_task_ms": 642340.9866666667, "avg_output_tokens_per_task": 17001.633333333335, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7555555555555555, "accuracy_std_dev": 0.02266234163693451, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.34417582888888887, "avg_duration_per_task_ms": 219931.23555555556, "avg_output_tokens_per_task": 11911.208888888888, "last_run_timestamp": "2026-10-03T08:23:27.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7555555, "accuracy_std_dev": 0.03119039105103792, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.0069151533333333326, "avg_duration_per_task_ms": 128985.71333333333, "avg_output_tokens_per_task": 4275.633333333333, "last_run_timestamp": "2026-10-03T08:51:15.407Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.75541675, "accuracy_std_dev": 0.033289157186499954, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0732536275, "avg_duration_per_task_ms": 407903.865, "avg_output_tokens_per_task": 9228.4025, "last_run_timestamp": "2026-10-03T16:19:32.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7552381428571429, "accuracy_std_dev": 0.032212325637550464, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07236689428571429, "avg_duration_per_task_ms": 290760.98, "avg_output_tokens_per_task": 11419.182857142858, "last_run_timestamp": "2026-10-03T16:21:33.429Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7552081741071428, "accuracy_std_dev": 0.03871863477403911, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.12397573883928571, "avg_duration_per_task_ms": 416217.65178571426, "avg_output_tokens_per_task": 20474.676339285714, "last_run_timestamp": "2026-10-03T16:14:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.755050595959596, "accuracy_std_dev": 0.048907866785416025, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.07672445454545455, "avg_duration_per_task_ms": 362896.6767676768, "avg_output_tokens_per_task": 12711.555555555555, "last_run_timestamp": "2026-10-03T16:20:30.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.755, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07160828000000001, "avg_duration_per_task_ms": 331234.47, "avg_output_tokens_per_task": 13566.125, "last_run_timestamp": "2026-10-03T08:43:45.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7549997500000001, "accuracy_std_dev": 0.022791366543880153, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0787181275, "avg_duration_per_task_ms": 246265.255, "avg_output_tokens_per_task": 9770.0025, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7549019380804953, "accuracy_std_dev": 0.022500075722423546, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.020211260061919506, "avg_duration_per_task_ms": 487189.79102167184, "avg_output_tokens_per_task": 17152.354489164085, "last_run_timestamp": "2026-10-03T08:34:10.987Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7548035989974938, "accuracy_std_dev": 0.028446815979286225, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.0686145037593985, "avg_duration_per_task_ms": 407411.1979949875, "avg_output_tokens_per_task": 10663.82456140351, "last_run_timestamp": "2026-10-08T23:43:42.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7546668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.19784133199999998, "avg_duration_per_task_ms": 328733.652, "avg_output_tokens_per_task": 12530.504, "last_run_timestamp": "2026-10-03T08:11:26.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7546666, "accuracy_std_dev": 0.0308113572151569, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.025647999999999997, "avg_duration_per_task_ms": 294521.988, "avg_output_tokens_per_task": 13284.328, "last_run_timestamp": "2026-10-03T16:12:20.035Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7545098823529411, "accuracy_std_dev": 0.035662554927909464, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.19809168117647058, "avg_duration_per_task_ms": 103146.24235294118, "avg_output_tokens_per_task": 7666.7011764705885, "last_run_timestamp": "2026-10-03T08:20:09.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7545043243243243, "accuracy_std_dev": 0.0263378748741976, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.023569141891891895, "avg_duration_per_task_ms": 316460.4864864865, "avg_output_tokens_per_task": 4132.989864864865, "last_run_timestamp": "2026-10-03T08:34:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.7544445, "accuracy_std_dev": 0.02678215501504689, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08861642333333332, "avg_duration_per_task_ms": 293005.13333333336, "avg_output_tokens_per_task": 12023.27, "last_run_timestamp": "2026-10-03T08:37:04.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7544445, "accuracy_std_dev": 0.025067718163340406, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008447913333333333, "avg_duration_per_task_ms": 189202.61333333334, "avg_output_tokens_per_task": 4437.176666666666, "last_run_timestamp": "2026-10-03T08:32:50.838Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7544443333333334, "accuracy_std_dev": 0.0343008332341294, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.009552396666666668, "avg_duration_per_task_ms": 212261.59, "avg_output_tokens_per_task": 15474.373333333333, "last_run_timestamp": "2026-10-03T16:05:34.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7544443333333334, "accuracy_std_dev": 0.02786659578340269, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008521756666666666, "avg_duration_per_task_ms": 190718.40666666668, "avg_output_tokens_per_task": 4439.933333333333, "last_run_timestamp": "2026-10-03T08:37:24.517Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.75416675, "accuracy_std_dev": 0.03169964729989123, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014154627500000003, "avg_duration_per_task_ms": 169480.67, "avg_output_tokens_per_task": 14972.985, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.75416675, "accuracy_std_dev": 0.012219036835507957, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1638202875, "avg_duration_per_task_ms": 453197.5425, "avg_output_tokens_per_task": 9528.3525, "last_run_timestamp": "2026-10-03T16:16:10.656Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.754074, "accuracy_std_dev": 0.039182303332556186, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19633914666666663, "avg_duration_per_task_ms": 104140.85111111111, "avg_output_tokens_per_task": 7619.131111111111, "last_run_timestamp": "2026-10-03T08:06:43.541Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.754, "accuracy_std_dev": 0.021385259722528513, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.022165832, "avg_duration_per_task_ms": 128895.106, "avg_output_tokens_per_task": 17780.99, "last_run_timestamp": "2026-10-03T08:11:39.228Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.753914899328859, "accuracy_std_dev": 0.040895046741628756, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.07723322147651007, "avg_duration_per_task_ms": 229349.62863534674, "avg_output_tokens_per_task": 14123.306487695749, "last_run_timestamp": "2026-10-03T08:50:19.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7538975456570156, "accuracy_std_dev": 0.023333857993043206, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.013686612472160357, "avg_duration_per_task_ms": 214524.49888641425, "avg_output_tokens_per_task": 16219.260579064588, "last_run_timestamp": "2026-10-03T08:34:35.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7538888333333333, "accuracy_std_dev": 0.019140560322838583, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.025353155, "avg_duration_per_task_ms": 294929.6716666667, "avg_output_tokens_per_task": 17480.86, "last_run_timestamp": "2026-10-03T08:34:34.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06322421, "avg_duration_per_task_ms": 228589.91, "avg_output_tokens_per_task": 10641.07, "last_run_timestamp": "2026-10-03T16:12:33.507Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7533335, "accuracy_std_dev": 0.01805358280000584, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15314664666666666, "avg_duration_per_task_ms": 312626.37, "avg_output_tokens_per_task": 20191.006666666668, "last_run_timestamp": "2026-10-03T08:43:11.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.7533335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01376083, "avg_duration_per_task_ms": 347095.91, "avg_output_tokens_per_task": 14466.76, "last_run_timestamp": "2026-10-09T02:54:47.806Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7533333999999999, "accuracy_std_dev": 0.015776240928687683, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.138680804, "avg_duration_per_task_ms": 204157.96, "avg_output_tokens_per_task": 8340.072, "last_run_timestamp": "2026-10-03T08:52:57.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.027755626724844277, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5350316133333334, "avg_duration_per_task_ms": 188978.14666666667, "avg_output_tokens_per_task": 10298.875555555556, "last_run_timestamp": "2026-10-03T08:09:23.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02739730544739352, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19939614444444445, "avg_duration_per_task_ms": 129460.44888888889, "avg_output_tokens_per_task": 7850.873333333333, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11451421333333334, "avg_duration_per_task_ms": 146387.56666666668, "avg_output_tokens_per_task": 17546.113333333335, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.29158895333333334, "avg_duration_per_task_ms": 171961.5, "avg_output_tokens_per_task": 10696.2, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06088320666666667, "avg_duration_per_task_ms": 1112758.12, "avg_output_tokens_per_task": 52398.04, "last_run_timestamp": "2026-10-03T16:14:15.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7533333, "accuracy_std_dev": 0.023475765282733617, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.021858692000000002, "avg_duration_per_task_ms": 119951.828, "avg_output_tokens_per_task": 15133.982, "last_run_timestamp": "2026-10-03T08:40:00.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7533332500000001, "accuracy_std_dev": 0.014907082584714572, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.243283165, "avg_duration_per_task_ms": 338011.175, "avg_output_tokens_per_task": 17182.585, "last_run_timestamp": "2026-10-03T08:32:03.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012521599999999999, "avg_duration_per_task_ms": 89897.54, "avg_output_tokens_per_task": 11032.16, "last_run_timestamp": "2026-10-08T06:07:32.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.032659863237109024, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12209446666666665, "avg_duration_per_task_ms": 147223.81333333332, "avg_output_tokens_per_task": 19831.306666666667, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2515843, "avg_duration_per_task_ms": 421006.06, "avg_output_tokens_per_task": 32340.48, "last_run_timestamp": "2026-10-01T08:00:24.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0196027, "avg_duration_per_task_ms": 76385.02, "avg_output_tokens_per_task": 9950, "last_run_timestamp": "2026-07-18T08:04:02.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07618375999999999, "avg_duration_per_task_ms": 574082.66, "avg_output_tokens_per_task": 14524.72, "last_run_timestamp": "2026-10-03T08:19:27.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7528571428571429, "accuracy_std_dev": 0.043949114567478, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1576592571428571, "avg_duration_per_task_ms": 349242.77428571426, "avg_output_tokens_per_task": 22490.9, "last_run_timestamp": "2026-10-03T08:08:18.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.03021638651133634, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5157013133333334, "avg_duration_per_task_ms": 178548.1511111111, "avg_output_tokens_per_task": 7818.084444444445, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.752499875, "accuracy_std_dev": 0.03850520617863741, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07637632, "avg_duration_per_task_ms": 229071.57, "avg_output_tokens_per_task": 9485.04, "last_run_timestamp": "2026-10-03T16:14:20.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7523456666666667, "accuracy_std_dev": 0.03581868589830568, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.22564089555555555, "avg_duration_per_task_ms": 98845.40814814816, "avg_output_tokens_per_task": 7557.676296296297, "last_run_timestamp": "2026-10-03T08:43:57.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7522221666666666, "accuracy_std_dev": 0.01511278994115765, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.44058496666666663, "avg_duration_per_task_ms": 458389.25, "avg_output_tokens_per_task": 35681.92333333333, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7519045714285715, "accuracy_std_dev": 0.03226166192184128, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.016922877142857146, "avg_duration_per_task_ms": 77003.04285714286, "avg_output_tokens_per_task": 4241.571428571428, "last_run_timestamp": "2026-10-03T08:48:31.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.751852, "accuracy_std_dev": 0.03366914875008805, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021374442222222224, "avg_duration_per_task_ms": 173854.87555555557, "avg_output_tokens_per_task": 14192.557777777778, "last_run_timestamp": "2026-10-03T08:32:03.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.7517005306122448, "accuracy_std_dev": 0.012796557796532647, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.09461716326530611, "avg_duration_per_task_ms": 209290.9224489796, "avg_output_tokens_per_task": 7253.518367346939, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.7513333999999999, "accuracy_std_dev": 0.030521588717496353, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.062658464, "avg_duration_per_task_ms": 1700876.012, "avg_output_tokens_per_task": 8868.604, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7511113333333332, "accuracy_std_dev": 0.014740562999952068, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.14565638333333333, "avg_duration_per_task_ms": 139837.15, "avg_output_tokens_per_task": 20029.903333333332, "last_run_timestamp": "2026-10-03T08:32:03.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7511113333333332, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48754586, "avg_duration_per_task_ms": 141651.29333333333, "avg_output_tokens_per_task": 7913.286666666667, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7510256923076923, "accuracy_std_dev": 0.024103851303776277, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.3703844146153846, "avg_duration_per_task_ms": 173913.4323076923, "avg_output_tokens_per_task": 11840.180769230768, "last_run_timestamp": "2026-10-03T08:17:09.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7508382676056339, "accuracy_std_dev": 0.0329423758499596, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.19496027364185115, "avg_duration_per_task_ms": 585792.7323943662, "avg_output_tokens_per_task": 27449.199195171026, "last_run_timestamp": "2026-10-03T08:06:43.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.75, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05066206, "avg_duration_per_task_ms": 368063.6, "avg_output_tokens_per_task": 8379.6, "last_run_timestamp": "2026-07-25T08:07:26.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.75, "accuracy_std_dev": 0.04384306609373936, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.50077502, "avg_duration_per_task_ms": 139265.845, "avg_output_tokens_per_task": 7986.47, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.75, "accuracy_std_dev": 0.025963164791681315, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012758686666666668, "avg_duration_per_task_ms": 426185.5333333333, "avg_output_tokens_per_task": 4133.86, "last_run_timestamp": "2026-10-09T04:18:40.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7499999, "accuracy_std_dev": 0.0217561485582812, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.03801625199999999, "avg_duration_per_task_ms": 294150.938, "avg_output_tokens_per_task": 23410.556, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7495237142857143, "accuracy_std_dev": 0.027338579222746164, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11086225999999999, "avg_duration_per_task_ms": 273444.3714285714, "avg_output_tokens_per_task": 12185.84, "last_run_timestamp": "2026-10-03T08:09:23.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7493332, "accuracy_std_dev": 0.011623875814890659, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.177717976, "avg_duration_per_task_ms": 166159.384, "avg_output_tokens_per_task": 12813.752, "last_run_timestamp": "2026-10-03T08:12:20.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7491665000000001, "accuracy_std_dev": 0.024480644282779816, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014165765000000002, "avg_duration_per_task_ms": 216043.245, "avg_output_tokens_per_task": 15278.05, "last_run_timestamp": "2026-10-03T16:07:05.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7490475714285715, "accuracy_std_dev": 0.017155993645346933, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034581794285714285, "avg_duration_per_task_ms": 410319.02857142856, "avg_output_tokens_per_task": 9408.788571428571, "last_run_timestamp": "2026-10-09T02:05:38.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7490367167630058, "accuracy_std_dev": 0.02179500605877504, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.011687421965317918, "avg_duration_per_task_ms": 205541.59826589594, "avg_output_tokens_per_task": 4094.841040462428, "last_run_timestamp": "2026-10-03T08:38:03.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.748889, "accuracy_std_dev": 0.015713326891527487, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013287926666666667, "avg_duration_per_task_ms": 103204.87333333334, "avg_output_tokens_per_task": 11122.173333333334, "last_run_timestamp": "2026-10-08T06:08:21.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.748889, "accuracy_std_dev": 0.019626097517789383, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.24296280222222222, "avg_duration_per_task_ms": 310167.38, "avg_output_tokens_per_task": 9997.322222222223, "last_run_timestamp": "2026-10-03T08:39:17.397Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7488737702702704, "accuracy_std_dev": 0.035681240826082414, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.3075426081081082, "avg_duration_per_task_ms": 149494.7972972973, "avg_output_tokens_per_task": 20582.41554054054, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7485673982808023, "accuracy_std_dev": 0.01859093254678699, "run_count": 14, "total_questions": 698, "avg_cost_per_task_usd": 0.006995356733524355, "avg_duration_per_task_ms": 195318.86962750717, "avg_output_tokens_per_task": 12847.305157593124, "last_run_timestamp": "2026-10-08T23:16:43.713Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7485381177944861, "accuracy_std_dev": 0.021900710861750612, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.028614501253132835, "avg_duration_per_task_ms": 631491.8922305765, "avg_output_tokens_per_task": 22158.218045112782, "last_run_timestamp": "2026-10-03T16:12:26.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7483332500000001, "accuracy_std_dev": 0.02803765820916874, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.338020715, "avg_duration_per_task_ms": 209359.69166666668, "avg_output_tokens_per_task": 11805.883333333333, "last_run_timestamp": "2026-10-03T08:02:27.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7481482222222222, "accuracy_std_dev": 0.03186030551146598, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5489024666666666, "avg_duration_per_task_ms": 183789.9622222222, "avg_output_tokens_per_task": 10558.302222222223, "last_run_timestamp": "2026-10-03T08:38:04.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.748, "accuracy_std_dev": 0.029333454545961698, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07858999599999998, "avg_duration_per_task_ms": 166655.096, "avg_output_tokens_per_task": 8640.336, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.022249863365997446, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020253733333333336, "avg_duration_per_task_ms": 168378.26333333334, "avg_output_tokens_per_task": 4203.816666666667, "last_run_timestamp": "2026-10-08T06:59:18.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7477776666666667, "accuracy_std_dev": 0.011166340517625106, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17740497333333335, "avg_duration_per_task_ms": 310994.1566666667, "avg_output_tokens_per_task": 10783.98, "last_run_timestamp": "2026-10-03T08:17:48.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7477776666666667, "accuracy_std_dev": 0.016063016338436865, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07122743666666666, "avg_duration_per_task_ms": 145473.51333333334, "avg_output_tokens_per_task": 4717.823333333334, "last_run_timestamp": "2026-10-03T08:39:53.772Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.747619, "accuracy_std_dev": 0.029477692093222922, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021878034285714287, "avg_duration_per_task_ms": 445632.1057142857, "avg_output_tokens_per_task": 15072.191428571428, "last_run_timestamp": "2026-10-03T16:20:50.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7476188571428571, "accuracy_std_dev": 0.023211532438045718, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11681679714285716, "avg_duration_per_task_ms": 334025.72571428574, "avg_output_tokens_per_task": 9744.797142857144, "last_run_timestamp": "2026-10-03T16:14:01.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7474074444444444, "accuracy_std_dev": 0.025809327224909778, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4897610111111111, "avg_duration_per_task_ms": 194827.99555555556, "avg_output_tokens_per_task": 10055.273333333333, "last_run_timestamp": "2026-10-03T08:17:06.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7470687587939698, "accuracy_std_dev": 0.027067095132974672, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.005836223618090453, "avg_duration_per_task_ms": 273469.26130653266, "avg_output_tokens_per_task": 12660.396984924622, "last_run_timestamp": "2026-10-03T16:13:42.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7470370000000001, "accuracy_std_dev": 0.028671978645437867, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.5621734177777777, "avg_duration_per_task_ms": 202509.61777777778, "avg_output_tokens_per_task": 9856.906666666666, "last_run_timestamp": "2026-10-03T08:21:15.324Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1787217, "avg_duration_per_task_ms": 84226.38, "avg_output_tokens_per_task": 6412.78, "last_run_timestamp": "2026-10-03T08:15:16.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02836242, "avg_duration_per_task_ms": 881974.84, "avg_output_tokens_per_task": 13870.46, "last_run_timestamp": "2026-07-18T08:03:38.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02698528, "avg_duration_per_task_ms": 147907.84, "avg_output_tokens_per_task": 11065.38, "last_run_timestamp": "2026-07-18T08:04:04.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07051216, "avg_duration_per_task_ms": 742250.02, "avg_output_tokens_per_task": 12898.82, "last_run_timestamp": "2026-10-08T23:33:36.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.38669936, "avg_duration_per_task_ms": 338232.82, "avg_output_tokens_per_task": 9439.24, "last_run_timestamp": "2026-07-18T08:04:09.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.027034372269390698, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015178880000000002, "avg_duration_per_task_ms": 145373.45333333334, "avg_output_tokens_per_task": 16435.604444444445, "last_run_timestamp": "2026-10-03T08:25:27.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.023726809342082992, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21412848666666662, "avg_duration_per_task_ms": 142371.99333333335, "avg_output_tokens_per_task": 22756.126666666667, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18399340666666666, "avg_duration_per_task_ms": 116870.40666666666, "avg_output_tokens_per_task": 5117.2266666666665, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.746666625, "accuracy_std_dev": 0.03349969319985448, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.087697375, "avg_duration_per_task_ms": 258190.44, "avg_output_tokens_per_task": 12176.68, "last_run_timestamp": "2026-10-03T08:20:48.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665555555556, "accuracy_std_dev": 0.012957518436208841, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3578504977777778, "avg_duration_per_task_ms": 113715.46888888889, "avg_output_tokens_per_task": 7110.575555555555, "last_run_timestamp": "2026-10-03T08:17:21.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.037271729999999996, "avg_duration_per_task_ms": 168918.11, "avg_output_tokens_per_task": 24042.94, "last_run_timestamp": "2026-10-03T08:36:42.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7466665, "accuracy_std_dev": 0.02854502616189613, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17205141333333332, "avg_duration_per_task_ms": 322261.24, "avg_output_tokens_per_task": 11142.893333333333, "last_run_timestamp": "2026-10-03T08:20:41.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7465427067238912, "accuracy_std_dev": 0.015385569261983973, "run_count": 14, "total_questions": 699, "avg_cost_per_task_usd": 0.048925171673819746, "avg_duration_per_task_ms": 263945.0042918455, "avg_output_tokens_per_task": 28450.080114449214, "last_run_timestamp": "2026-10-05T17:03:10.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7462311155778895, "accuracy_std_dev": 0.026033403023507623, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.031126798994974874, "avg_duration_per_task_ms": 552773.9422110553, "avg_output_tokens_per_task": 14499.40703517588, "last_run_timestamp": "2026-10-04T00:00:28.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.745926, "accuracy_std_dev": 0.02095136179610079, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3237732711111111, "avg_duration_per_task_ms": 198551.82222222222, "avg_output_tokens_per_task": 11463.67111111111, "last_run_timestamp": "2026-10-03T08:12:20.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.745833375, "accuracy_std_dev": 0.053274708014069624, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08002613749999998, "avg_duration_per_task_ms": 231305.2225, "avg_output_tokens_per_task": 9750.2375, "last_run_timestamp": "2026-10-03T16:12:22.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.74583325, "accuracy_std_dev": 0.04047453074388262, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11309086500000001, "avg_duration_per_task_ms": 292417.6425, "avg_output_tokens_per_task": 9222.4525, "last_run_timestamp": "2026-10-03T16:18:39.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7455556666666666, "accuracy_std_dev": 0.02506781419913237, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.3528521766666667, "avg_duration_per_task_ms": 231482.565, "avg_output_tokens_per_task": 10841.498333333333, "last_run_timestamp": "2026-10-03T08:06:43.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7455555, "accuracy_std_dev": 0.01739159368536039, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.014102986666666668, "avg_duration_per_task_ms": 390602.61333333334, "avg_output_tokens_per_task": 16178.07, "last_run_timestamp": "2026-10-08T23:19:45.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7451851111111111, "accuracy_std_dev": 0.023471128692475918, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01492189111111111, "avg_duration_per_task_ms": 122706.42666666667, "avg_output_tokens_per_task": 15563.857777777777, "last_run_timestamp": "2026-10-03T13:25:39.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.745, "accuracy_std_dev": 0.019649399202011284, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.019549249999999997, "avg_duration_per_task_ms": 83357.6, "avg_output_tokens_per_task": 4168.05, "last_run_timestamp": "2026-10-03T08:30:42.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7448975714285715, "accuracy_std_dev": 0.011805499999999969, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.26889934693877554, "avg_duration_per_task_ms": 528124.887755102, "avg_output_tokens_per_task": 13421.112244897959, "last_run_timestamp": "2026-10-03T08:42:17.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7448486363636363, "accuracy_std_dev": 0.04090809875423143, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.021699527272727274, "avg_duration_per_task_ms": 329455.08181818185, "avg_output_tokens_per_task": 12517.08909090909, "last_run_timestamp": "2026-10-03T08:11:53.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7447619999999999, "accuracy_std_dev": 0.026836107941140986, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3472446428571429, "avg_duration_per_task_ms": 201649.75142857144, "avg_output_tokens_per_task": 11858.474285714286, "last_run_timestamp": "2026-10-03T08:17:06.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.7447619999999999, "accuracy_std_dev": 0.02512556764959325, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02136247142857143, "avg_duration_per_task_ms": 216896.61714285714, "avg_output_tokens_per_task": 12936.834285714285, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7447618571428571, "accuracy_std_dev": 0.01621859463198753, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05547540571428572, "avg_duration_per_task_ms": 220891.45428571428, "avg_output_tokens_per_task": 9176.125714285714, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.02665859574468085, "avg_duration_per_task_ms": 538486.1063829787, "avg_output_tokens_per_task": 9135.659574468085, "last_run_timestamp": "2026-10-03T08:11:53.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7446235483870968, "accuracy_std_dev": 0.021113320137770862, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.14085775806451614, "avg_duration_per_task_ms": 569919.2116935484, "avg_output_tokens_per_task": 27271.122983870966, "last_run_timestamp": "2026-10-03T16:05:51.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7444445, "accuracy_std_dev": 0.03235589896113743, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007050623333333332, "avg_duration_per_task_ms": 202444.91666666666, "avg_output_tokens_per_task": 4335.703333333333, "last_run_timestamp": "2026-10-03T08:42:41.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7444444444444445, "accuracy_std_dev": 0.028803274848959327, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5183121866666667, "avg_duration_per_task_ms": 226393.7577777778, "avg_output_tokens_per_task": 10022.293333333333, "last_run_timestamp": "2026-10-03T08:15:30.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2319787333333332, "avg_duration_per_task_ms": 98531.56, "avg_output_tokens_per_task": 5524.42, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.34168689999999996, "avg_duration_per_task_ms": 130878.06, "avg_output_tokens_per_task": 7080.526666666667, "last_run_timestamp": "2026-10-03T08:24:43.704Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.02845816509467109, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00811217, "avg_duration_per_task_ms": 250640.82333333333, "avg_output_tokens_per_task": 5287.056666666666, "last_run_timestamp": "2026-10-03T08:42:16.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7442424545454546, "accuracy_std_dev": 0.04942282999763578, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08388880545454544, "avg_duration_per_task_ms": 185197.90181818182, "avg_output_tokens_per_task": 8027.018181818182, "last_run_timestamp": "2026-10-03T16:25:24.079Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7441947955056181, "accuracy_std_dev": 0.013929720069329346, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.10463494606741572, "avg_duration_per_task_ms": 943614.8921348314, "avg_output_tokens_per_task": 21227.357303370787, "last_run_timestamp": "2026-09-26T08:24:43.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.74416675, "accuracy_std_dev": 0.006400770358909954, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08563862, "avg_duration_per_task_ms": 309336.805, "avg_output_tokens_per_task": 9480.84, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.744, "accuracy_std_dev": 0.03855146055858323, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.047344880000000006, "avg_duration_per_task_ms": 518226.236, "avg_output_tokens_per_task": 9205.268, "last_run_timestamp": "2026-08-29T16:10:33.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7438095714285714, "accuracy_std_dev": 0.03184416704990163, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08113430285714286, "avg_duration_per_task_ms": 191849.82857142857, "avg_output_tokens_per_task": 9843.122857142856, "last_run_timestamp": "2026-10-03T16:14:23.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.7438094285714286, "accuracy_std_dev": 0.015474279737285213, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008623625714285715, "avg_duration_per_task_ms": 150720.6542857143, "avg_output_tokens_per_task": 13331.717142857144, "last_run_timestamp": "2026-10-03T16:14:52.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.743333375, "accuracy_std_dev": 0.019436727836350823, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.3260761375, "avg_duration_per_task_ms": 567167.5325, "avg_output_tokens_per_task": 26014.51, "last_run_timestamp": "2026-10-03T16:09:05.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.743333375, "accuracy_std_dev": 0.038151762010218804, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0479863775, "avg_duration_per_task_ms": 502843.075, "avg_output_tokens_per_task": 9263.875, "last_run_timestamp": "2026-10-04T00:00:36.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.030368011314333716, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.073473185, "avg_duration_per_task_ms": 258433.37, "avg_output_tokens_per_task": 9863.935, "last_run_timestamp": "2026-10-03T16:21:25.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7433330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34579436, "avg_duration_per_task_ms": 133736.57, "avg_output_tokens_per_task": 7063.35, "last_run_timestamp": "2026-10-03T08:17:21.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7428572857142857, "accuracy_std_dev": 0.030207079708071675, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.28986799428571425, "avg_duration_per_task_ms": 509147.33714285714, "avg_output_tokens_per_task": 9527.642857142857, "last_run_timestamp": "2026-10-03T16:04:32.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7425475772357722, "accuracy_std_dev": 0.026593049538554243, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.05760138211382114, "avg_duration_per_task_ms": 427993.5081300813, "avg_output_tokens_per_task": 12666.333333333334, "last_run_timestamp": "2026-10-03T08:41:56.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.74250025, "accuracy_std_dev": 0.024874694303397962, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01637653, "avg_duration_per_task_ms": 283609.14, "avg_output_tokens_per_task": 16652.845, "last_run_timestamp": "2026-10-03T08:46:48.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7425, "accuracy_std_dev": 0.020258742970628775, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.010828945, "avg_duration_per_task_ms": 216783.6, "avg_output_tokens_per_task": 14322.1425, "last_run_timestamp": "2026-10-03T16:11:33.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7425, "accuracy_std_dev": 0.023318376160230372, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07715312749999999, "avg_duration_per_task_ms": 252677.67, "avg_output_tokens_per_task": 11166.4925, "last_run_timestamp": "2026-10-03T16:11:33.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.74239996, "accuracy_std_dev": 0.0339871010313972, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.3750872199999999, "avg_duration_per_task_ms": 202842.4192, "avg_output_tokens_per_task": 11926.788, "last_run_timestamp": "2026-10-03T08:42:18.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7423934827586207, "accuracy_std_dev": 0.04075671767696707, "run_count": 10, "total_questions": 493, "avg_cost_per_task_usd": 0.11995382961460448, "avg_duration_per_task_ms": 491199.4949290061, "avg_output_tokens_per_task": 23541.371196754564, "last_run_timestamp": "2026-10-03T08:18:33.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.006285707880227596, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.38184122000000004, "avg_duration_per_task_ms": 149071.25333333333, "avg_output_tokens_per_task": 12794.173333333334, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0028365266666666666, "avg_duration_per_task_ms": 68565.14666666667, "avg_output_tokens_per_task": 11675.026666666667, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7422222222222222, "accuracy_std_dev": 0.023306958640323044, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08909326444444444, "avg_duration_per_task_ms": 804074.62, "avg_output_tokens_per_task": 37512.27111111111, "last_run_timestamp": "2026-10-03T16:15:37.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7422221111111111, "accuracy_std_dev": 0.02012307808597685, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009145446666666668, "avg_duration_per_task_ms": 281815.08222222223, "avg_output_tokens_per_task": 4191.742222222222, "last_run_timestamp": "2026-10-03T08:37:33.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7421122716049383, "accuracy_std_dev": 0.04949233809146621, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.07618280658436215, "avg_duration_per_task_ms": 605963.6255144033, "avg_output_tokens_per_task": 7686.909465020576, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7419048571428571, "accuracy_std_dev": 0.03181571083343471, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07701604857142856, "avg_duration_per_task_ms": 557828.82, "avg_output_tokens_per_task": 11690.26, "last_run_timestamp": "2026-10-03T16:12:13.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7419047142857143, "accuracy_std_dev": 0.02753696505589889, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.34922034571428573, "avg_duration_per_task_ms": 221709.42857142858, "avg_output_tokens_per_task": 10867.765714285715, "last_run_timestamp": "2026-10-03T08:22:39.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7416665, "accuracy_std_dev": 0.015898969219732467, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011650780000000001, "avg_duration_per_task_ms": 547605.185, "avg_output_tokens_per_task": 5560.365, "last_run_timestamp": "2026-10-03T08:36:39.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7413794367816092, "accuracy_std_dev": 0.07093542318941382, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.2491148678160919, "avg_duration_per_task_ms": 429289.867816092, "avg_output_tokens_per_task": 27858.181034482757, "last_run_timestamp": "2026-10-03T16:11:17.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7413334, "accuracy_std_dev": 0.04420155150715866, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.13771882, "avg_duration_per_task_ms": 864032.39, "avg_output_tokens_per_task": 32740.022, "last_run_timestamp": "2026-10-03T16:09:32.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.7412120909090909, "accuracy_std_dev": 0.04686698917236574, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12050856545454547, "avg_duration_per_task_ms": 159418.54, "avg_output_tokens_per_task": 9545.092727272728, "last_run_timestamp": "2026-10-03T16:12:33.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7411615606060606, "accuracy_std_dev": 0.022128401427803163, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.11188127777777777, "avg_duration_per_task_ms": 105277.7601010101, "avg_output_tokens_per_task": 9824.214646464647, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7409522857142857, "accuracy_std_dev": 0.029905244158911024, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11990698857142856, "avg_duration_per_task_ms": 408401.10285714286, "avg_output_tokens_per_task": 9217.857142857143, "last_run_timestamp": "2026-10-03T16:14:05.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7403703333333334, "accuracy_std_dev": 0.030325141571529955, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3409674066666667, "avg_duration_per_task_ms": 128710.22, "avg_output_tokens_per_task": 5246.528888888889, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7402943534136547, "accuracy_std_dev": 0.02302499266101948, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.06053981927710843, "avg_duration_per_task_ms": 161020.49799196786, "avg_output_tokens_per_task": 6902.349397590361, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400002, "accuracy_std_dev": 0.01264900523203309, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034869184, "avg_duration_per_task_ms": 129959.044, "avg_output_tokens_per_task": 16349.368, "last_run_timestamp": "2026-10-03T08:21:01.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7400001428571428, "accuracy_std_dev": 0.03454918738936282, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12511439714285713, "avg_duration_per_task_ms": 410478.8742857143, "avg_output_tokens_per_task": 11800.674285714285, "last_run_timestamp": "2026-10-03T16:34:12.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.74, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3455118266666667, "avg_duration_per_task_ms": 100747.32666666666, "avg_output_tokens_per_task": 6984.213333333333, "last_run_timestamp": "2026-10-03T08:20:22.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2696536, "avg_duration_per_task_ms": 310259.6, "avg_output_tokens_per_task": 11435.58, "last_run_timestamp": "2026-07-18T08:04:29.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05110574, "avg_duration_per_task_ms": 158677.44, "avg_output_tokens_per_task": 8249.06, "last_run_timestamp": "2026-07-18T16:00:09.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.74, "accuracy_std_dev": 0.044555500139339316, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.0935073933333332, "avg_duration_per_task_ms": 295552.94, "avg_output_tokens_per_task": 22860.126666666667, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02561014, "avg_duration_per_task_ms": 147210.62, "avg_output_tokens_per_task": 10787.18, "last_run_timestamp": "2026-07-18T08:04:42.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01199366, "avg_duration_per_task_ms": 376489.52, "avg_output_tokens_per_task": 20183.24, "last_run_timestamp": "2026-10-07T21:32:26.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.74, "accuracy_std_dev": 0.0354338036978815, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09767998, "avg_duration_per_task_ms": 450597.1575, "avg_output_tokens_per_task": 9690.325, "last_run_timestamp": "2026-10-03T16:05:51.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12237688000000001, "avg_duration_per_task_ms": 90244.42, "avg_output_tokens_per_task": 9936.14, "last_run_timestamp": "2026-07-18T08:04:45.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01290664, "avg_duration_per_task_ms": 114271.64, "avg_output_tokens_per_task": 4375.16, "last_run_timestamp": "2026-10-09T04:02:01.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03319066, "avg_duration_per_task_ms": 77162.14, "avg_output_tokens_per_task": 4453.24, "last_run_timestamp": "2026-10-04T03:00:16.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7399999090909091, "accuracy_std_dev": 0.025266298870790318, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.023259952727272725, "avg_duration_per_task_ms": 396353.68727272726, "avg_output_tokens_per_task": 12260.443636363636, "last_run_timestamp": "2026-10-03T08:16:24.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7399998750000001, "accuracy_std_dev": 0.02185807758265525, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0728203575, "avg_duration_per_task_ms": 345522.1125, "avg_output_tokens_per_task": 12908.68, "last_run_timestamp": "2026-10-03T16:14:15.894Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7399998333333334, "accuracy_std_dev": 0.02108185106844802, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.015687136666666667, "avg_duration_per_task_ms": 84185.48333333334, "avg_output_tokens_per_task": 4443.856666666667, "last_run_timestamp": "2026-10-03T08:49:53.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7396624683544304, "accuracy_std_dev": 0.02347519769052809, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.019082217721518985, "avg_duration_per_task_ms": 310670.8101265823, "avg_output_tokens_per_task": 15967.20506329114, "last_run_timestamp": "2026-10-03T08:37:28.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7396296666666666, "accuracy_std_dev": 0.019177150471908547, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.48443006888888884, "avg_duration_per_task_ms": 673286.5122222222, "avg_output_tokens_per_task": 9807.494444444445, "last_run_timestamp": "2026-10-03T08:24:07.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.7392593333333333, "accuracy_std_dev": 0.014553267185679581, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015606891111111108, "avg_duration_per_task_ms": 184036.77333333335, "avg_output_tokens_per_task": 4234.36, "last_run_timestamp": "2026-10-03T13:17:20.799Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7392472258064516, "accuracy_std_dev": 0.020530217177614083, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.04685905645161291, "avg_duration_per_task_ms": 881254.7661290322, "avg_output_tokens_per_task": 14245.16935483871, "last_run_timestamp": "2026-10-09T01:09:41.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7391550136986301, "accuracy_std_dev": 0.013576518152268981, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.0136173698630137, "avg_duration_per_task_ms": 57786.109589041094, "avg_output_tokens_per_task": 3732.6301369863013, "last_run_timestamp": "2026-10-03T08:21:31.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7388888333333333, "accuracy_std_dev": 0.030164102613408253, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06139170666666667, "avg_duration_per_task_ms": 280149.0466666667, "avg_output_tokens_per_task": 9746.213333333333, "last_run_timestamp": "2026-10-05T19:59:12.757Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.03154897514486578, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03057054888888889, "avg_duration_per_task_ms": 269640.66222222225, "avg_output_tokens_per_task": 17038.204444444444, "last_run_timestamp": "2026-10-03T08:35:45.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.017992647486688756, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3103866155555556, "avg_duration_per_task_ms": 191198.25333333333, "avg_output_tokens_per_task": 10879.702222222222, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7384840804020101, "accuracy_std_dev": 0.03165357517370423, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.10895770226130652, "avg_duration_per_task_ms": 110251.2148241206, "avg_output_tokens_per_task": 8058.163316582914, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7382865983935742, "accuracy_std_dev": 0.027461257286584682, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.07498479919678715, "avg_duration_per_task_ms": 91080.29718875502, "avg_output_tokens_per_task": 4367.955823293173, "last_run_timestamp": "2026-09-26T08:11:26.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7381818181818182, "accuracy_std_dev": 0.02622221420793717, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.023726459999999998, "avg_duration_per_task_ms": 166279.69454545455, "avg_output_tokens_per_task": 7657.0654545454545, "last_run_timestamp": "2026-10-03T16:10:46.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7381758445945945, "accuracy_std_dev": 0.015122946755951125, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.020533699324324327, "avg_duration_per_task_ms": 105846.9831081081, "avg_output_tokens_per_task": 3919.314189189189, "last_run_timestamp": "2026-10-03T08:06:57.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7380951428571428, "accuracy_std_dev": 0.030594855288470453, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12595829142857143, "avg_duration_per_task_ms": 255162.36285714287, "avg_output_tokens_per_task": 10029.605714285713, "last_run_timestamp": "2026-10-03T16:12:24.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7377778333333334, "accuracy_std_dev": 0.02484511527723086, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06635720666666667, "avg_duration_per_task_ms": 282495.5133333333, "avg_output_tokens_per_task": 8158.293333333333, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.737500125, "accuracy_std_dev": 0.01854046812675925, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.386765285, "avg_duration_per_task_ms": 179778.535, "avg_output_tokens_per_task": 10745.6125, "last_run_timestamp": "2026-10-03T08:40:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7374686954887218, "accuracy_std_dev": 0.031950697455106813, "run_count": 11, "total_questions": 532, "avg_cost_per_task_usd": 0.0578316992481203, "avg_duration_per_task_ms": 124019.84022556391, "avg_output_tokens_per_task": 7638.883458646616, "last_run_timestamp": "2026-10-03T16:12:10.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7371428571428571, "accuracy_std_dev": 0.033210782885032, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4983071857142858, "avg_duration_per_task_ms": 191159.05714285714, "avg_output_tokens_per_task": 9902.92, "last_run_timestamp": "2026-10-03T08:21:29.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7369696363636363, "accuracy_std_dev": 0.0327047385954244, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.027384705454545456, "avg_duration_per_task_ms": 418295.4309090909, "avg_output_tokens_per_task": 12820.870909090909, "last_run_timestamp": "2026-10-03T08:16:37.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7369696363636363, "accuracy_std_dev": 0.041813872418074854, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09632115999999999, "avg_duration_per_task_ms": 149694.9690909091, "avg_output_tokens_per_task": 9129.581818181818, "last_run_timestamp": "2026-10-03T16:15:37.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.37755702999999996, "avg_duration_per_task_ms": 287090.13, "avg_output_tokens_per_task": 9995.37, "last_run_timestamp": "2026-10-03T08:10:45.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2879983000000002, "avg_duration_per_task_ms": 310547.73, "avg_output_tokens_per_task": 22913.4, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06833741, "avg_duration_per_task_ms": 266166.56, "avg_output_tokens_per_task": 13745.94, "last_run_timestamp": "2026-10-03T08:19:53.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7366668333333333, "accuracy_std_dev": 0.023333261905247794, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.35079288666666664, "avg_duration_per_task_ms": 89188.23666666666, "avg_output_tokens_per_task": 6914.113333333334, "last_run_timestamp": "2026-10-03T08:12:46.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7366668, "accuracy_std_dev": 0.022900761755015906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.063241744, "avg_duration_per_task_ms": 1530537.112, "avg_output_tokens_per_task": 25222.716, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7366667499999999, "accuracy_std_dev": 0.0328295429291591, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10887684250000002, "avg_duration_per_task_ms": 323590.875, "avg_output_tokens_per_task": 18425.0325, "last_run_timestamp": "2026-10-03T08:19:27.960Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.736666625, "accuracy_std_dev": 0.03915774721219767, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0416237425, "avg_duration_per_task_ms": 215980.8525, "avg_output_tokens_per_task": 20025.32, "last_run_timestamp": "2026-10-03T08:21:01.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7366665, "accuracy_std_dev": 0.023333357143440252, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.019450683333333333, "avg_duration_per_task_ms": 216938.31, "avg_output_tokens_per_task": 4564.706666666667, "last_run_timestamp": "2026-10-03T08:34:35.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7365900057471265, "accuracy_std_dev": 0.030044845419567656, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.09514021264367815, "avg_duration_per_task_ms": 383890.3362068966, "avg_output_tokens_per_task": 11244.488505747126, "last_run_timestamp": "2026-10-03T16:13:47.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.736, "accuracy_std_dev": 0.01717892766152766, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.23393872400000001, "avg_duration_per_task_ms": 250693.228, "avg_output_tokens_per_task": 12185.72, "last_run_timestamp": "2026-10-03T08:11:53.179Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.736, "accuracy_std_dev": 0.006798758063058283, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.091687124, "avg_duration_per_task_ms": 467869.472, "avg_output_tokens_per_task": 12583.56, "last_run_timestamp": "2026-10-03T08:09:52.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.736, "accuracy_std_dev": 0.03542738426133094, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.42442627200000005, "avg_duration_per_task_ms": 39355.976, "avg_output_tokens_per_task": 9202.272, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7358334999999999, "accuracy_std_dev": 0.025097105854062156, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015653559999999997, "avg_duration_per_task_ms": 644029.88, "avg_output_tokens_per_task": 23956.73, "last_run_timestamp": "2026-10-03T08:19:27.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.73583325, "accuracy_std_dev": 0.04026825845424036, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0081671125, "avg_duration_per_task_ms": 333111.9, "avg_output_tokens_per_task": 15655.8425, "last_run_timestamp": "2026-10-03T08:38:11.814Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13620342666666665, "avg_duration_per_task_ms": 101042.81333333334, "avg_output_tokens_per_task": 8769.966666666667, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7355554444444444, "accuracy_std_dev": 0.012171531247693556, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3527382688888889, "avg_duration_per_task_ms": 87063.18222222223, "avg_output_tokens_per_task": 7051.828888888889, "last_run_timestamp": "2026-10-03T08:18:17.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7353034451858913, "accuracy_std_dev": 0.02679997092672807, "run_count": 21, "total_questions": 1049, "avg_cost_per_task_usd": 0.1235421267874166, "avg_duration_per_task_ms": 175822.2707340324, "avg_output_tokens_per_task": 9155.737845567206, "last_run_timestamp": "2026-10-03T16:13:53.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7352381428571428, "accuracy_std_dev": 0.03375250188156883, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05847352000000001, "avg_duration_per_task_ms": 654076.3628571428, "avg_output_tokens_per_task": 14818.84, "last_run_timestamp": "2026-10-04T00:00:34.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7352381428571428, "accuracy_std_dev": 0.031615541975556306, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.047925131428571424, "avg_duration_per_task_ms": 368302.0857142857, "avg_output_tokens_per_task": 10123.34857142857, "last_run_timestamp": "2026-10-03T16:15:19.157Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.735194730964467, "accuracy_std_dev": 0.02636439034355808, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.11292960406091371, "avg_duration_per_task_ms": 484722.62436548225, "avg_output_tokens_per_task": 29978.527918781725, "last_run_timestamp": "2026-10-03T08:25:13.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7351928099173552, "accuracy_std_dev": 0.026445667029780136, "run_count": 10, "total_questions": 484, "avg_cost_per_task_usd": 0.019751508264462807, "avg_duration_per_task_ms": 93530.47314049587, "avg_output_tokens_per_task": 6382.072314049587, "last_run_timestamp": "2026-10-03T16:12:29.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7349997500000001, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12295462999999998, "avg_duration_per_task_ms": 89986.35, "avg_output_tokens_per_task": 23350.75, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7346668, "accuracy_std_dev": 0.020177018832325044, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.067634488, "avg_duration_per_task_ms": 150581.522, "avg_output_tokens_per_task": 9458.92, "last_run_timestamp": "2026-10-05T17:43:17.994Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7346666000000001, "accuracy_std_dev": 0.025785270559759498, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.382410584, "avg_duration_per_task_ms": 202893.328, "avg_output_tokens_per_task": 8639.172, "last_run_timestamp": "2026-10-03T08:18:05.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7345953452115812, "accuracy_std_dev": 0.020421322845629582, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.01083965478841871, "avg_duration_per_task_ms": 494548.15144766145, "avg_output_tokens_per_task": 14266.481069042316, "last_run_timestamp": "2026-10-03T08:38:09.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7342569118387909, "accuracy_std_dev": 0.03174336773937983, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.07517244332493701, "avg_duration_per_task_ms": 212080.59697733, "avg_output_tokens_per_task": 8222.277078085643, "last_run_timestamp": "2026-10-03T16:14:02.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7340741111111111, "accuracy_std_dev": 0.04903445388804455, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.00916024, "avg_duration_per_task_ms": 334894.9577777778, "avg_output_tokens_per_task": 15526.06, "last_run_timestamp": "2026-10-03T08:11:53.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7339392727272728, "accuracy_std_dev": 0.021733921472251252, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.007870807272727274, "avg_duration_per_task_ms": 338594.0509090909, "avg_output_tokens_per_task": 14023.138181818182, "last_run_timestamp": "2026-10-03T08:21:29.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7333336666666668, "accuracy_std_dev": 0.03771236166328251, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.44470873333333333, "avg_duration_per_task_ms": 130557.96, "avg_output_tokens_per_task": 20380.893333333333, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.7333335714285714, "accuracy_std_dev": 0.023904731551109535, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019937122857142855, "avg_duration_per_task_ms": 479765.5571428572, "avg_output_tokens_per_task": 14687.234285714285, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7333332727272728, "accuracy_std_dev": 0.022201865425683618, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07251096727272727, "avg_duration_per_task_ms": 663094.6745454546, "avg_output_tokens_per_task": 8454.441818181818, "last_run_timestamp": "2026-10-03T16:16:13.260Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.73333325, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025873325, "avg_duration_per_task_ms": 147169.53, "avg_output_tokens_per_task": 18134.845, "last_run_timestamp": "2026-10-03T08:34:36.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.73333325, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17039939499999998, "avg_duration_per_task_ms": 114141.445, "avg_output_tokens_per_task": 5083.04, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.016329931618554536, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012738966666666665, "avg_duration_per_task_ms": 70118.46666666666, "avg_output_tokens_per_task": 10694.486666666666, "last_run_timestamp": "2026-10-08T06:08:43.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04628359, "avg_duration_per_task_ms": 154349.4, "avg_output_tokens_per_task": 38415.97, "last_run_timestamp": "2026-10-03T08:10:45.808Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02503509736024079, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06745117142857142, "avg_duration_per_task_ms": 76562.92857142857, "avg_output_tokens_per_task": 8496.594285714285, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7325925555555556, "accuracy_std_dev": 0.02443313150262758, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.3673924444444444, "avg_duration_per_task_ms": 230445.1311111111, "avg_output_tokens_per_task": 10853.041111111112, "last_run_timestamp": "2026-10-03T08:12:59.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.732381, "accuracy_std_dev": 0.01034532550064454, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08187367142857144, "avg_duration_per_task_ms": 300173.6142857143, "avg_output_tokens_per_task": 13347.99142857143, "last_run_timestamp": "2026-10-03T16:10:32.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7320002, "accuracy_std_dev": 0.03435759143129796, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.016945259999999997, "avg_duration_per_task_ms": 153216.504, "avg_output_tokens_per_task": 14926.164, "last_run_timestamp": "2026-10-03T08:34:50.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7319933969849246, "accuracy_std_dev": 0.06840954592516678, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.07448894472361808, "avg_duration_per_task_ms": 91970.93467336684, "avg_output_tokens_per_task": 13988.37688442211, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.73166675, "accuracy_std_dev": 0.01280170344866263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021476995, "avg_duration_per_task_ms": 208483.135, "avg_output_tokens_per_task": 8547.82, "last_run_timestamp": "2026-10-03T16:05:47.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73166675, "accuracy_std_dev": 0.015184321229067165, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.29879598, "avg_duration_per_task_ms": 177541.565, "avg_output_tokens_per_task": 10870.785, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7315437382550335, "accuracy_std_dev": 0.03437878873504547, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.014048604026845638, "avg_duration_per_task_ms": 181440.27516778524, "avg_output_tokens_per_task": 3805.51677852349, "last_run_timestamp": "2026-10-03T08:39:48.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7315152727272727, "accuracy_std_dev": 0.033162046414239345, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009558798181818184, "avg_duration_per_task_ms": 364447.56727272726, "avg_output_tokens_per_task": 14249.385454545454, "last_run_timestamp": "2026-10-03T08:10:45.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.731515090909091, "accuracy_std_dev": 0.023799046212571194, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02329288181818182, "avg_duration_per_task_ms": 89903.6490909091, "avg_output_tokens_per_task": 7512.998181818181, "last_run_timestamp": "2026-10-03T16:13:58.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7313987589285714, "accuracy_std_dev": 0.03616424235329221, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.08183186607142856, "avg_duration_per_task_ms": 746817.6116071428, "avg_output_tokens_per_task": 14329.484375, "last_run_timestamp": "2026-10-05T20:34:16.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7313334, "accuracy_std_dev": 0.11123746291443364, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.14579965199999997, "avg_duration_per_task_ms": 618150.212, "avg_output_tokens_per_task": 27105.312, "last_run_timestamp": "2026-10-03T08:15:42.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.731293, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.12389479591836736, "avg_duration_per_task_ms": 269993.51020408166, "avg_output_tokens_per_task": 23771, "last_run_timestamp": "2026-07-18T08:00:15.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7311113333333332, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10536965333333334, "avg_duration_per_task_ms": 88766.42666666667, "avg_output_tokens_per_task": 4017.0533333333333, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.022662199996175744, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3988358066666667, "avg_duration_per_task_ms": 172929.82666666666, "avg_output_tokens_per_task": 27148.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.019116342955701554, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4446010933333333, "avg_duration_per_task_ms": 752160.54, "avg_output_tokens_per_task": 9652.03, "last_run_timestamp": "2026-10-08T22:43:35.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7308334999999999, "accuracy_std_dev": 0.023731723962872993, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.043652735000000005, "avg_duration_per_task_ms": 134583.575, "avg_output_tokens_per_task": 27937.025, "last_run_timestamp": "2026-10-03T08:15:00.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.730303, "accuracy_std_dev": 0.02244879491400172, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07475869636363637, "avg_duration_per_task_ms": 189076.56181818183, "avg_output_tokens_per_task": 8589.47090909091, "last_run_timestamp": "2026-10-03T16:12:07.465Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7300587002518891, "accuracy_std_dev": 0.027075268354798905, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.37323221158690173, "avg_duration_per_task_ms": 401791.3123425693, "avg_output_tokens_per_task": 9265.304785894206, "last_run_timestamp": "2026-10-03T08:39:26.414Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.730000125, "accuracy_std_dev": 0.04988867885461965, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07825786500000001, "avg_duration_per_task_ms": 212986.7025, "avg_output_tokens_per_task": 8293.9625, "last_run_timestamp": "2026-09-19T08:12:24.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.012764169999999998, "avg_duration_per_task_ms": 81526.21, "avg_output_tokens_per_task": 10652.99, "last_run_timestamp": "2026-10-08T06:07:22.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.73, "accuracy_std_dev": 0.03241608458334048, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07427336285714285, "avg_duration_per_task_ms": 226239.70857142858, "avg_output_tokens_per_task": 11230.525714285714, "last_run_timestamp": "2026-10-03T16:17:46.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.73, "accuracy_std_dev": 0.036666636365138984, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3388476266666666, "avg_duration_per_task_ms": 404706.78, "avg_output_tokens_per_task": 9319.38, "last_run_timestamp": "2026-10-03T08:15:30.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7299516521739131, "accuracy_std_dev": 0.017388525374334593, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.020714037681159418, "avg_duration_per_task_ms": 279014.0231884058, "avg_output_tokens_per_task": 10794.56811594203, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7295238571428572, "accuracy_std_dev": 0.034215143421598336, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.3555786895238095, "avg_duration_per_task_ms": 206423.8238095238, "avg_output_tokens_per_task": 10296.156190476191, "last_run_timestamp": "2026-10-03T08:16:52.730Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7293334, "accuracy_std_dev": 0.013727396775791089, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.21736060000000001, "avg_duration_per_task_ms": 395127.28, "avg_output_tokens_per_task": 9443.484, "last_run_timestamp": "2026-10-03T08:37:59.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7291667620967742, "accuracy_std_dev": 0.06506350304902127, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.1482808629032258, "avg_duration_per_task_ms": 1327131.8165322582, "avg_output_tokens_per_task": 27401.167338709678, "last_run_timestamp": "2026-10-03T16:07:44.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7290600769230768, "accuracy_std_dev": 0.010018378923258995, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.1270635282051282, "avg_duration_per_task_ms": 193981.8564102564, "avg_output_tokens_per_task": 10184.36923076923, "last_run_timestamp": "2026-10-03T16:07:34.513Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7289057819548871, "accuracy_std_dev": 0.03087378380499384, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.06965342355889724, "avg_duration_per_task_ms": 217209.81203007518, "avg_output_tokens_per_task": 11564.82456140351, "last_run_timestamp": "2026-10-03T16:23:41.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.728889, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06727646666666667, "avg_duration_per_task_ms": 501138.4733333333, "avg_output_tokens_per_task": 9493.433333333332, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.728889, "accuracy_std_dev": 0.052305021830285715, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024965873333333333, "avg_duration_per_task_ms": 476087.61333333334, "avg_output_tokens_per_task": 18580.853333333333, "last_run_timestamp": "2026-10-03T08:29:04.085Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.7285712857142858, "accuracy_std_dev": 0.02217236334277612, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05063508571428572, "avg_duration_per_task_ms": 392975.47428571427, "avg_output_tokens_per_task": 14780.671428571428, "last_run_timestamp": "2026-10-04T00:00:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7284848181818182, "accuracy_std_dev": 0.02815423218248235, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07946297454545455, "avg_duration_per_task_ms": 222298.18181818182, "avg_output_tokens_per_task": 8681.263636363636, "last_run_timestamp": "2026-10-03T16:14:07.111Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7283335, "accuracy_std_dev": 0.023746461725907702, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05840676, "avg_duration_per_task_ms": 311422.2825, "avg_output_tokens_per_task": 9732.815, "last_run_timestamp": "2026-10-04T00:00:36.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.72833325, "accuracy_std_dev": 0.022298511411919422, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.044024975, "avg_duration_per_task_ms": 136603.385, "avg_output_tokens_per_task": 25927.71, "last_run_timestamp": "2026-10-03T08:49:47.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.727878909090909, "accuracy_std_dev": 0.03385259734752348, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.022492889090909095, "avg_duration_per_task_ms": 121138.31272727273, "avg_output_tokens_per_task": 7028.689090909091, "last_run_timestamp": "2026-10-03T16:15:50.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7278787272727273, "accuracy_std_dev": 0.027970785626470766, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02821320545454546, "avg_duration_per_task_ms": 208497.22363636363, "avg_output_tokens_per_task": 8992.374545454546, "last_run_timestamp": "2026-10-03T16:24:32.945Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7278566673427991, "accuracy_std_dev": 0.03572243647191496, "run_count": 10, "total_questions": 493, "avg_cost_per_task_usd": 0.10080718052738337, "avg_duration_per_task_ms": 324790.75456389453, "avg_output_tokens_per_task": 14787.588235294117, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.72750025, "accuracy_std_dev": 0.014790298059454364, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013181285000000001, "avg_duration_per_task_ms": 37840.265, "avg_output_tokens_per_task": 3828.01, "last_run_timestamp": "2026-10-03T08:06:57.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7275, "accuracy_std_dev": 0.011273272229038026, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.25819418250000004, "avg_duration_per_task_ms": 292629.7475, "avg_output_tokens_per_task": 10249.3025, "last_run_timestamp": "2026-09-26T08:16:17.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7272728080808081, "accuracy_std_dev": 0.03401092910650194, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12744196464646465, "avg_duration_per_task_ms": 652406.3232323233, "avg_output_tokens_per_task": 27175.661616161615, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7271427142857143, "accuracy_std_dev": 0.03945770950094827, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.00689044, "avg_duration_per_task_ms": 394490.1285714286, "avg_output_tokens_per_task": 14238.282857142856, "last_run_timestamp": "2026-10-03T08:19:00.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7270693489932887, "accuracy_std_dev": 0.056280655844911175, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.2920954026845638, "avg_duration_per_task_ms": 417994.4194630872, "avg_output_tokens_per_task": 10865.342281879195, "last_run_timestamp": "2026-10-03T08:36:28.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11300302000000001, "avg_duration_per_task_ms": 353321.78, "avg_output_tokens_per_task": 3658.14, "last_run_timestamp": "2026-10-03T00:00:06.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7266667, "accuracy_std_dev": 0.04152637180888791, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.12686799, "avg_duration_per_task_ms": 183230.478, "avg_output_tokens_per_task": 12462.162, "last_run_timestamp": "2026-10-03T16:12:35.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.050625568127402036, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17683866, "avg_duration_per_task_ms": 285023.12333333335, "avg_output_tokens_per_task": 10017.94, "last_run_timestamp": "2026-10-03T08:23:53.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7266665, "accuracy_std_dev": 0.033665214491370776, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027464944999999998, "avg_duration_per_task_ms": 167583.24, "avg_output_tokens_per_task": 16249.64, "last_run_timestamp": "2026-10-03T08:11:53.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7266665, "accuracy_std_dev": 0.03162256578378802, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.325251515, "avg_duration_per_task_ms": 113433.705, "avg_output_tokens_per_task": 10925.445, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7264311717171718, "accuracy_std_dev": 0.023522929859766603, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04122346464646465, "avg_duration_per_task_ms": 213769.9797979798, "avg_output_tokens_per_task": 25500.383838383837, "last_run_timestamp": "2026-10-03T08:37:32.589Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.725714, "accuracy_std_dev": 0.027004721179823374, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04600672857142857, "avg_duration_per_task_ms": 172219.76, "avg_output_tokens_per_task": 8290.077142857142, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7254901363636364, "accuracy_std_dev": 0.03196228807593238, "run_count": 15, "total_questions": 748, "avg_cost_per_task_usd": 0.11537986497326203, "avg_duration_per_task_ms": 109572.1256684492, "avg_output_tokens_per_task": 9160.804812834225, "last_run_timestamp": "2026-10-08T07:00:33.849Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7251207652173913, "accuracy_std_dev": 0.03173202093295975, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.020070034782608694, "avg_duration_per_task_ms": 306185.0115942029, "avg_output_tokens_per_task": 7045.43768115942, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7248995421686747, "accuracy_std_dev": 0.02769605902019275, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.043781132530120476, "avg_duration_per_task_ms": 222582.8293172691, "avg_output_tokens_per_task": 9978.564257028112, "last_run_timestamp": "2026-10-03T16:16:50.387Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7247618571428571, "accuracy_std_dev": 0.04843106328860957, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04376539714285715, "avg_duration_per_task_ms": 289052.61714285717, "avg_output_tokens_per_task": 14151.488571428572, "last_run_timestamp": "2026-10-03T08:24:26.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.7247618571428571, "accuracy_std_dev": 0.02776646862894973, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05254494857142857, "avg_duration_per_task_ms": 272263.16, "avg_output_tokens_per_task": 8601.185714285713, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7247618571428571, "accuracy_std_dev": 0.03161563234689254, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034889062857142855, "avg_duration_per_task_ms": 356175.16285714286, "avg_output_tokens_per_task": 8342.902857142857, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7246665, "accuracy_std_dev": 0.02459454772607134, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.070817994, "avg_duration_per_task_ms": 195208.492, "avg_output_tokens_per_task": 8421.906, "last_run_timestamp": "2026-10-03T16:25:05.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.7244897346938776, "accuracy_std_dev": 0.009027500000000022, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.08190116326530611, "avg_duration_per_task_ms": 1045226.0918367347, "avg_output_tokens_per_task": 15985.285714285714, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.7244446666666666, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05245255333333333, "avg_duration_per_task_ms": 276706.37333333335, "avg_output_tokens_per_task": 8448.833333333334, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7242422727272728, "accuracy_std_dev": 0.021700055647565487, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0786679290909091, "avg_duration_per_task_ms": 259230.04545454544, "avg_output_tokens_per_task": 9112.834545454545, "last_run_timestamp": "2026-10-03T16:06:47.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.724, "accuracy_std_dev": 0.028315782800410083, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.034526184, "avg_duration_per_task_ms": 154106.95, "avg_output_tokens_per_task": 8565.566, "last_run_timestamp": "2026-10-03T16:08:50.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7238494100418409, "accuracy_std_dev": 0.024050891639188762, "run_count": 5, "total_questions": 239, "avg_cost_per_task_usd": 0.1394956820083682, "avg_duration_per_task_ms": 649605.6820083682, "avg_output_tokens_per_task": 27120.7949790795, "last_run_timestamp": "2026-10-08T23:41:49.599Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7238092857142858, "accuracy_std_dev": 0.03648997874451985, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06535770571428572, "avg_duration_per_task_ms": 383455.27142857143, "avg_output_tokens_per_task": 11487.757142857143, "last_run_timestamp": "2026-10-03T16:10:02.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7237036666666666, "accuracy_std_dev": 0.03501115714867033, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3328492511111112, "avg_duration_per_task_ms": 204813.59333333332, "avg_output_tokens_per_task": 10311.322222222223, "last_run_timestamp": "2026-10-03T08:11:53.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7237035555555555, "accuracy_std_dev": 0.03371788875396401, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.29463488, "avg_duration_per_task_ms": 344302.6911111111, "avg_output_tokens_per_task": 9396.468888888889, "last_run_timestamp": "2026-10-03T08:36:28.986Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7236843198380566, "accuracy_std_dev": 0.04197719449284337, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.02163497165991903, "avg_duration_per_task_ms": 102577.22267206477, "avg_output_tokens_per_task": 6911.8421052631575, "last_run_timestamp": "2026-10-03T16:04:32.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.72333325, "accuracy_std_dev": 0.01374358436462263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011429815, "avg_duration_per_task_ms": 101515.345, "avg_output_tokens_per_task": 5183.985, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12502769, "avg_duration_per_task_ms": 371456.84, "avg_output_tokens_per_task": 16328.88, "last_run_timestamp": "2026-10-03T16:15:26.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7226666, "accuracy_std_dev": 0.02619600308902105, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.065411432, "avg_duration_per_task_ms": 217655.416, "avg_output_tokens_per_task": 9279.2, "last_run_timestamp": "2026-10-03T16:12:07.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7225001249999999, "accuracy_std_dev": 0.02623366025947152, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0554660425, "avg_duration_per_task_ms": 314525.895, "avg_output_tokens_per_task": 11294.4225, "last_run_timestamp": "2026-10-03T16:05:51.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7225, "accuracy_std_dev": 0.017618787323479452, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10703709, "avg_duration_per_task_ms": 327457.75, "avg_output_tokens_per_task": 14810.6125, "last_run_timestamp": "2026-10-04T00:00:34.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7224880669856459, "accuracy_std_dev": 0.03220370562155287, "run_count": 21, "total_questions": 1045, "avg_cost_per_task_usd": 0.11458385071770336, "avg_duration_per_task_ms": 139702.72918660287, "avg_output_tokens_per_task": 8362.703349282297, "last_run_timestamp": "2026-10-09T22:51:18.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7224242727272727, "accuracy_std_dev": 0.03827310252072574, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.047232909090909096, "avg_duration_per_task_ms": 150332.00363636363, "avg_output_tokens_per_task": 8525.521818181818, "last_run_timestamp": "2026-10-08T22:58:29.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.031427125187575726, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009912693333333332, "avg_duration_per_task_ms": 115398.83333333333, "avg_output_tokens_per_task": 5230.286666666667, "last_run_timestamp": "2026-10-03T08:11:53.182Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7222221428571428, "accuracy_std_dev": 0.024312166707227047, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.046834047619047615, "avg_duration_per_task_ms": 58108.6462585034, "avg_output_tokens_per_task": 13243.625850340137, "last_run_timestamp": "2026-09-29T21:00:05.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7222221111111111, "accuracy_std_dev": 0.027917975508621698, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.13745050505050505, "avg_duration_per_task_ms": 712939.898989899, "avg_output_tokens_per_task": 27938.80808080808, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.722222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019033145833333334, "avg_duration_per_task_ms": 65993.54166666667, "avg_output_tokens_per_task": 3355.9791666666665, "last_run_timestamp": "2026-09-12T08:12:22.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.72166675, "accuracy_std_dev": 0.018782342477377516, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0501920125, "avg_duration_per_task_ms": 390041.62, "avg_output_tokens_per_task": 9293.87, "last_run_timestamp": "2026-10-03T16:12:08.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7214285714285714, "accuracy_std_dev": 0.03730415283414796, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.057019652857142854, "avg_duration_per_task_ms": 116171.83142857143, "avg_output_tokens_per_task": 8636.995714285715, "last_run_timestamp": "2026-10-03T16:13:38.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7211088376753507, "accuracy_std_dev": 0.03778148726612018, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.1016803647294589, "avg_duration_per_task_ms": 212461.24448897794, "avg_output_tokens_per_task": 10204.877755511023, "last_run_timestamp": "2026-10-03T16:20:53.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7207406666666667, "accuracy_std_dev": 0.027477417091131398, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3540747222222222, "avg_duration_per_task_ms": 237714.0088888889, "avg_output_tokens_per_task": 10830.737777777778, "last_run_timestamp": "2026-10-03T08:11:26.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7205384141414142, "accuracy_std_dev": 0.051112796809483045, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 1.6761988181818182, "avg_duration_per_task_ms": 313695.66161616164, "avg_output_tokens_per_task": 23404.97474747475, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03553636, "avg_duration_per_task_ms": 263015.66, "avg_output_tokens_per_task": 14384.46, "last_run_timestamp": "2026-07-18T16:00:09.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.72, "accuracy_std_dev": 0.021081956479416224, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03251169, "avg_duration_per_task_ms": 105065.11, "avg_output_tokens_per_task": 22739.925, "last_run_timestamp": "2026-10-03T09:03:15.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.72, "accuracy_std_dev": 0.03231762505197435, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.36211610499999997, "avg_duration_per_task_ms": 150104.23, "avg_output_tokens_per_task": 23123.4, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.72, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.037352619999999996, "avg_duration_per_task_ms": 206767.2, "avg_output_tokens_per_task": 11957.14, "last_run_timestamp": "2026-09-26T08:10:43.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7199996666666667, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09322211333333333, "avg_duration_per_task_ms": 149908.11333333334, "avg_output_tokens_per_task": 49346.81333333333, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.719620806020067, "accuracy_std_dev": 0.02809879417737037, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.05551290969899665, "avg_duration_per_task_ms": 968941.5852842809, "avg_output_tokens_per_task": 14053.80602006689, "last_run_timestamp": "2026-10-04T00:00:34.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.719409382278481, "accuracy_std_dev": 0.05128756107661487, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.06101479240506329, "avg_duration_per_task_ms": 936708.475949367, "avg_output_tokens_per_task": 64773.12151898734, "last_run_timestamp": "2026-10-03T16:09:19.692Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7193332, "accuracy_std_dev": 0.03143937799893632, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045152332, "avg_duration_per_task_ms": 326196.928, "avg_output_tokens_per_task": 13427.836, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7193332, "accuracy_std_dev": 0.02332387046268266, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011747176000000002, "avg_duration_per_task_ms": 276612.664, "avg_output_tokens_per_task": 17492.112, "last_run_timestamp": "2026-10-03T08:34:32.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7192392617449664, "accuracy_std_dev": 0.027973240079682514, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.06318058053691275, "avg_duration_per_task_ms": 493682.4899328859, "avg_output_tokens_per_task": 14513.312080536913, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7191666250000001, "accuracy_std_dev": 0.02633896432539395, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.300338065, "avg_duration_per_task_ms": 406405.855, "avg_output_tokens_per_task": 9208.345, "last_run_timestamp": "2026-10-03T08:11:53.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7191665, "accuracy_std_dev": 0.008291511728870679, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.046859950000000004, "avg_duration_per_task_ms": 203049.32, "avg_output_tokens_per_task": 32440.35, "last_run_timestamp": "2026-10-03T08:38:29.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.7188888333333333, "accuracy_std_dev": 0.026504201858175022, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3197561666666667, "avg_duration_per_task_ms": 193449.66, "avg_output_tokens_per_task": 11623.543333333333, "last_run_timestamp": "2026-10-03T08:06:43.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7188888333333333, "accuracy_std_dev": 0.03297943429581869, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11799081000000002, "avg_duration_per_task_ms": 673829.7433333333, "avg_output_tokens_per_task": 29526.56, "last_run_timestamp": "2026-10-03T08:37:23.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7188551919191919, "accuracy_std_dev": 0.027525589315898408, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.29904142424242425, "avg_duration_per_task_ms": 154139.7323232323, "avg_output_tokens_per_task": 5036.570707070707, "last_run_timestamp": "2026-10-01T19:05:12.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.718667, "accuracy_std_dev": 0.015999999999999966, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012890876, "avg_duration_per_task_ms": 931505.22, "avg_output_tokens_per_task": 16944.504, "last_run_timestamp": "2026-10-01T09:55:07.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7184344343434343, "accuracy_std_dev": 0.09707188521882894, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.07242429797979798, "avg_duration_per_task_ms": 168379.42676767678, "avg_output_tokens_per_task": 14277.45202020202, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.71833325, "accuracy_std_dev": 0.022298399296530245, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.039712805, "avg_duration_per_task_ms": 138975.945, "avg_output_tokens_per_task": 15594.74, "last_run_timestamp": "2026-10-03T08:38:07.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7181818181818181, "accuracy_std_dev": 0.03494188642672075, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03435519272727272, "avg_duration_per_task_ms": 183390.90545454546, "avg_output_tokens_per_task": 9125.847272727273, "last_run_timestamp": "2026-10-03T16:08:50.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7180952857142857, "accuracy_std_dev": 0.033752526067220005, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.13201746285714283, "avg_duration_per_task_ms": 330154.5142857143, "avg_output_tokens_per_task": 10313.082857142857, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7177649598853869, "accuracy_std_dev": 0.02734697248773434, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.13113841547277938, "avg_duration_per_task_ms": 74329.58739255014, "avg_output_tokens_per_task": 10317.49570200573, "last_run_timestamp": "2026-08-29T17:11:32.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7173333000000001, "accuracy_std_dev": 0.03878137353691847, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.102133462, "avg_duration_per_task_ms": 118936.744, "avg_output_tokens_per_task": 9103.426, "last_run_timestamp": "2026-10-03T16:12:20.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.7170371111111111, "accuracy_std_dev": 0.016956299841680886, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12728777111111111, "avg_duration_per_task_ms": 101323.44222222222, "avg_output_tokens_per_task": 3885.56, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015077799999999999, "avg_duration_per_task_ms": 476878.72, "avg_output_tokens_per_task": 26097.54, "last_run_timestamp": "2026-09-28T09:00:02.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.716666625, "accuracy_std_dev": 0.019148607435904436, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0257953375, "avg_duration_per_task_ms": 222790.8875, "avg_output_tokens_per_task": 10050.0125, "last_run_timestamp": "2026-10-03T16:12:13.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.716666625, "accuracy_std_dev": 0.02309402881123982, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06049543999999999, "avg_duration_per_task_ms": 304582.9575, "avg_output_tokens_per_task": 9615.66, "last_run_timestamp": "2026-10-04T00:00:28.270Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06700715, "avg_duration_per_task_ms": 992682.5, "avg_output_tokens_per_task": 40930.55, "last_run_timestamp": "2026-10-03T08:22:39.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7164444666666666, "accuracy_std_dev": 0.014780646674471161, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.03797310133333333, "avg_duration_per_task_ms": 188817.49466666667, "avg_output_tokens_per_task": 9923.881333333333, "last_run_timestamp": "2026-10-03T16:15:22.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7155558333333333, "accuracy_std_dev": 0.0289741860421575, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04852732, "avg_duration_per_task_ms": 969869.3766666667, "avg_output_tokens_per_task": 8596.123333333333, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.040855070557058414, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012743326666666667, "avg_duration_per_task_ms": 1224143.2066666668, "avg_output_tokens_per_task": 18669.653333333332, "last_run_timestamp": "2026-08-29T09:20:05.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7155553333333334, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13937678, "avg_duration_per_task_ms": 263819.3466666667, "avg_output_tokens_per_task": 10311.9, "last_run_timestamp": "2026-10-03T16:18:50.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.715488212121212, "accuracy_std_dev": 0.013403360380889571, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04391666666666666, "avg_duration_per_task_ms": 58944.35858585859, "avg_output_tokens_per_task": 8110.530303030303, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.715238, "accuracy_std_dev": 0.036811758695907394, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0628034857142857, "avg_duration_per_task_ms": 75005.29714285715, "avg_output_tokens_per_task": 9080.965714285714, "last_run_timestamp": "2026-09-26T08:24:48.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.715, "accuracy_std_dev": 0.029202378139459823, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09407665500000001, "avg_duration_per_task_ms": 83159.84, "avg_output_tokens_per_task": 3660.855, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7148146666666667, "accuracy_std_dev": 0.031075936024161484, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09082762666666666, "avg_duration_per_task_ms": 156907.88888888888, "avg_output_tokens_per_task": 48272.70222222222, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.714166625, "accuracy_std_dev": 0.0311691275781722, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0320175475, "avg_duration_per_task_ms": 366948.7475, "avg_output_tokens_per_task": 5303.03, "last_run_timestamp": "2026-10-03T08:46:44.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7139999, "accuracy_std_dev": 0.1569839316620972, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.059771752, "avg_duration_per_task_ms": 154448.396, "avg_output_tokens_per_task": 18970.404, "last_run_timestamp": "2026-10-03T16:07:44.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7129095028901734, "accuracy_std_dev": 0.03295967347643888, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.05974209248554914, "avg_duration_per_task_ms": 481297.3583815029, "avg_output_tokens_per_task": 14073.28612716763, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7126667, "accuracy_std_dev": 0.028666690698613956, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.00856998, "avg_duration_per_task_ms": 314663.846, "avg_output_tokens_per_task": 14195.494, "last_run_timestamp": "2026-10-03T08:22:39.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.712622768382353, "accuracy_std_dev": 0.07553839113396205, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.0459969705882353, "avg_duration_per_task_ms": 260039.11948529413, "avg_output_tokens_per_task": 26119.363970588234, "last_run_timestamp": "2026-10-03T16:03:45.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.7121428571428572, "accuracy_std_dev": 0.03208647153253476, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.13476882285714287, "avg_duration_per_task_ms": 139447.84285714285, "avg_output_tokens_per_task": 8900.078571428572, "last_run_timestamp": "2026-10-03T16:12:20.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.712121101010101, "accuracy_std_dev": 0.029111223892169177, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12218062626262625, "avg_duration_per_task_ms": 537818.2474747475, "avg_output_tokens_per_task": 55422.06060606061, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.71166675, "accuracy_std_dev": 0.019075733568790994, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07509325, "avg_duration_per_task_ms": 322731.32, "avg_output_tokens_per_task": 33483.435, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7113343768844221, "accuracy_std_dev": 0.031502220499778535, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.40403888609715255, "avg_duration_per_task_ms": 235745.01172529312, "avg_output_tokens_per_task": 9241.058626465661, "last_run_timestamp": "2026-10-03T08:37:32.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7113334, "accuracy_std_dev": 0.032048543736650496, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.113843728, "avg_duration_per_task_ms": 97247.482, "avg_output_tokens_per_task": 9868.37, "last_run_timestamp": "2026-10-03T16:12:20.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7113332, "accuracy_std_dev": 0.04279658761116358, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.09086514600000001, "avg_duration_per_task_ms": 216121.818, "avg_output_tokens_per_task": 8339.134, "last_run_timestamp": "2026-10-03T16:13:56.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.011331263428624743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07186812, "avg_duration_per_task_ms": 348295.26, "avg_output_tokens_per_task": 33945.17333333333, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.7104762857142857, "accuracy_std_dev": 0.03359069896229476, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04053524, "avg_duration_per_task_ms": 377429.66285714286, "avg_output_tokens_per_task": 9822.331428571428, "last_run_timestamp": "2026-10-03T16:14:22.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7103825300546448, "accuracy_std_dev": 0.03582559506394764, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.07252062477231329, "avg_duration_per_task_ms": 252602.8378870674, "avg_output_tokens_per_task": 8810.581056466302, "last_run_timestamp": "2026-10-03T16:18:22.330Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7103705555555555, "accuracy_std_dev": 0.030809822424065814, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03036840888888889, "avg_duration_per_task_ms": 105409.86666666667, "avg_output_tokens_per_task": 5076.06, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12391315, "avg_duration_per_task_ms": 76197.77, "avg_output_tokens_per_task": 4134.025, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.71, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.32462002, "avg_duration_per_task_ms": 110744.1, "avg_output_tokens_per_task": 8803.62, "last_run_timestamp": "2026-10-06T00:00:08.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7099996666666667, "accuracy_std_dev": 0.03144660377352204, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.24028676333333332, "avg_duration_per_task_ms": 296408.56666666665, "avg_output_tokens_per_task": 20949.47, "last_run_timestamp": "2026-10-03T08:39:24.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.7097435384615385, "accuracy_std_dev": 0.030226114342543606, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.11513952769230772, "avg_duration_per_task_ms": 89188.81846153847, "avg_output_tokens_per_task": 10509.030769230769, "last_run_timestamp": "2026-10-03T16:12:29.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.70916675, "accuracy_std_dev": 0.01730050176114842, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0730223075, "avg_duration_per_task_ms": 288375.1675, "avg_output_tokens_per_task": 11030.785, "last_run_timestamp": "2026-10-03T16:18:18.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7091114702886249, "accuracy_std_dev": 0.03584624085968895, "run_count": 24, "total_questions": 1178, "avg_cost_per_task_usd": 0.09626076570458404, "avg_duration_per_task_ms": 76008.29456706283, "avg_output_tokens_per_task": 8032.944821731749, "last_run_timestamp": "2026-10-03T16:16:10.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.01749773883169545, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11639628666666667, "avg_duration_per_task_ms": 755791.72, "avg_output_tokens_per_task": 27174.273333333334, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7088121666666666, "accuracy_std_dev": 0.09574853521360532, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07333836494252874, "avg_duration_per_task_ms": 189816.08620689655, "avg_output_tokens_per_task": 23421.635057471263, "last_run_timestamp": "2026-10-03T16:12:22.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.70833325, "accuracy_std_dev": 0.014043632407162335, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.026613565000000002, "avg_duration_per_task_ms": 223836.8925, "avg_output_tokens_per_task": 10131.55, "last_run_timestamp": "2026-10-03T16:10:17.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7079999, "accuracy_std_dev": 0.061665789544041345, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.29473372800000003, "avg_duration_per_task_ms": 421854.93, "avg_output_tokens_per_task": 9864.05, "last_run_timestamp": "2026-10-05T20:05:10.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08614948, "avg_duration_per_task_ms": 52133.28, "avg_output_tokens_per_task": 7649.3, "last_run_timestamp": "2026-07-18T08:04:46.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.22511376, "avg_duration_per_task_ms": 2937291.8, "avg_output_tokens_per_task": 55713.12, "last_run_timestamp": "2026-10-03T08:06:57.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7066666363636364, "accuracy_std_dev": 0.03527669560198404, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08133863090909091, "avg_duration_per_task_ms": 74950.49272727272, "avg_output_tokens_per_task": 8535.285454545454, "last_run_timestamp": "2026-10-03T16:09:05.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.7066665000000001, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.054561849999999995, "avg_duration_per_task_ms": 281726.22, "avg_output_tokens_per_task": 8888.58, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7058335, "accuracy_std_dev": 0.044526811524069405, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.069417755, "avg_duration_per_task_ms": 340857.2225, "avg_output_tokens_per_task": 10007.65, "last_run_timestamp": "2026-10-04T00:00:36.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7056452258064517, "accuracy_std_dev": 0.012405393094940622, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.6654760282258064, "avg_duration_per_task_ms": 686538.5927419355, "avg_output_tokens_per_task": 55521.342741935485, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.7051926733668341, "accuracy_std_dev": 0.031064655778060283, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.04506834422110553, "avg_duration_per_task_ms": 454167.2688442211, "avg_output_tokens_per_task": 9105.824120603014, "last_run_timestamp": "2026-10-03T16:26:46.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7049549256756757, "accuracy_std_dev": 0.09115185938022449, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.009106506756756757, "avg_duration_per_task_ms": 215462.97972972973, "avg_output_tokens_per_task": 3961.837837837838, "last_run_timestamp": "2026-10-05T20:02:53.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.028782815862246673, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.059492744, "avg_duration_per_task_ms": 531413.344, "avg_output_tokens_per_task": 14170.264, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7036621748878924, "accuracy_std_dev": 0.10398250962233035, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.19077936547085203, "avg_duration_per_task_ms": 1129697.726457399, "avg_output_tokens_per_task": 51230.17264573991, "last_run_timestamp": "2026-10-03T16:07:05.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.701111, "accuracy_std_dev": 0.011811220865487748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13736028333333333, "avg_duration_per_task_ms": 494538.36333333334, "avg_output_tokens_per_task": 57774.223333333335, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7011108333333333, "accuracy_std_dev": 0.020608011147906124, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.051610086666666666, "avg_duration_per_task_ms": 398260.51, "avg_output_tokens_per_task": 9237.883333333333, "last_run_timestamp": "2026-10-03T16:12:07.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.7000003333333334, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010915493333333333, "avg_duration_per_task_ms": 115952.82, "avg_output_tokens_per_task": 5345.16, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7000003333333332, "accuracy_std_dev": 0.024944382578492925, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12734470666666667, "avg_duration_per_task_ms": 591806.9066666666, "avg_output_tokens_per_task": 57532.753333333334, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.70000025, "accuracy_std_dev": 0.03162288201109283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12919751, "avg_duration_per_task_ms": 506623.765, "avg_output_tokens_per_task": 10892.11, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.019795359999999998, "avg_duration_per_task_ms": 396615.94, "avg_output_tokens_per_task": 23218.96, "last_run_timestamp": "2026-10-01T08:03:06.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01539542, "avg_duration_per_task_ms": 2335452.5, "avg_output_tokens_per_task": 25378.6, "last_run_timestamp": "2026-07-18T08:04:08.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0112321, "avg_duration_per_task_ms": 596563.34, "avg_output_tokens_per_task": 4097.46, "last_run_timestamp": "2026-09-12T09:21:58.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.6998645487804879, "accuracy_std_dev": 0.05531710950158186, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.031559408536585364, "avg_duration_per_task_ms": 1019512.3414634146, "avg_output_tokens_per_task": 43385.81707317073, "last_run_timestamp": "2026-10-03T16:10:32.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.699494803030303, "accuracy_std_dev": 0.010638522711559854, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.021464873737373737, "avg_duration_per_task_ms": 737254.1363636364, "avg_output_tokens_per_task": 19210.790404040403, "last_run_timestamp": "2026-10-08T22:48:13.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.6986666, "accuracy_std_dev": 0.007483243510671025, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04888268, "avg_duration_per_task_ms": 227205.572, "avg_output_tokens_per_task": 6367.684, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6985507420289855, "accuracy_std_dev": 0.027723151851321442, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.01719656231884058, "avg_duration_per_task_ms": 309641.62898550724, "avg_output_tokens_per_task": 9142.672463768116, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6977778333333333, "accuracy_std_dev": 0.026573791972647694, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04297996, "avg_duration_per_task_ms": 184061.30666666667, "avg_output_tokens_per_task": 8458.556666666667, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6977778333333333, "accuracy_std_dev": 0.017915990515148434, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21024033333333336, "avg_duration_per_task_ms": 335029.48333333334, "avg_output_tokens_per_task": 34166.46666666667, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.6974275704697986, "accuracy_std_dev": 0.019759375021746003, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.01971821476510067, "avg_duration_per_task_ms": 421936.07718120806, "avg_output_tokens_per_task": 9528.71476510067, "last_run_timestamp": "2026-10-03T16:05:51.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.6972790204081633, "accuracy_std_dev": 0.04361100000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.08154698979591836, "avg_duration_per_task_ms": 436567.05102040817, "avg_output_tokens_per_task": 13545.377551020409, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14710045000000002, "avg_duration_per_task_ms": 163276.98, "avg_output_tokens_per_task": 5053.91, "last_run_timestamp": "2026-10-01T08:02:59.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.6961111666666666, "accuracy_std_dev": 0.025416492410878035, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19325967666666666, "avg_duration_per_task_ms": 1865316.5633333332, "avg_output_tokens_per_task": 13131.506666666666, "last_run_timestamp": "2026-10-04T00:00:37.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04339576, "avg_duration_per_task_ms": 106333.2, "avg_output_tokens_per_task": 10497.62, "last_run_timestamp": "2026-10-03T16:12:20.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.693333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08382193, "avg_duration_per_task_ms": 858486.8, "avg_output_tokens_per_task": 15204.12, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.060288180000000004, "avg_duration_per_task_ms": 350636.38, "avg_output_tokens_per_task": 10540.6, "last_run_timestamp": "2026-07-18T08:04:08.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02187568, "avg_duration_per_task_ms": 236451.76, "avg_output_tokens_per_task": 11760.74, "last_run_timestamp": "2026-07-18T08:04:03.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6931599132947976, "accuracy_std_dev": 0.025748486861030352, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.033708676300578035, "avg_duration_per_task_ms": 745817.6300578035, "avg_output_tokens_per_task": 19785.147398843932, "last_run_timestamp": "2026-10-03T16:26:51.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6929823609022557, "accuracy_std_dev": 0.026094683447675196, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.03234121052631579, "avg_duration_per_task_ms": 241069.54887218046, "avg_output_tokens_per_task": 9648.423558897242, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6929165, "accuracy_std_dev": 0.02385937097609657, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.10850378125, "avg_duration_per_task_ms": 77457.025, "avg_output_tokens_per_task": 9370.97875, "last_run_timestamp": "2026-10-03T16:16:14.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.6927083333333334, "accuracy_std_dev": 0.006303999999999976, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.03757610416666667, "avg_duration_per_task_ms": 270206.4791666667, "avg_output_tokens_per_task": 15460, "last_run_timestamp": "2026-10-05T08:30:52.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.6923811428571428, "accuracy_std_dev": 0.04719813017612506, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03026138, "avg_duration_per_task_ms": 311692.76857142855, "avg_output_tokens_per_task": 10090.888571428572, "last_run_timestamp": "2026-10-03T16:07:02.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6923076113360324, "accuracy_std_dev": 0.004408577938519381, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.009435951417004049, "avg_duration_per_task_ms": 259361.54655870446, "avg_output_tokens_per_task": 9080.562753036438, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.6914287142857143, "accuracy_std_dev": 0.13703043242466176, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03953962285714285, "avg_duration_per_task_ms": 274567.7942857143, "avg_output_tokens_per_task": 10410.422857142858, "last_run_timestamp": "2026-10-03T16:12:22.581Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.690000125, "accuracy_std_dev": 0.05343730230709045, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0988822075, "avg_duration_per_task_ms": 253655.9675, "avg_output_tokens_per_task": 10551.77, "last_run_timestamp": "2026-10-03T16:11:17.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.69, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01726069, "avg_duration_per_task_ms": 298145.74, "avg_output_tokens_per_task": 10516.76, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6891892297297297, "accuracy_std_dev": 0.03170486152346707, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6622049662162162, "avg_duration_per_task_ms": 777791.8783783783, "avg_output_tokens_per_task": 59355.4527027027, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6879166250000001, "accuracy_std_dev": 0.07218680699569953, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05169213000000001, "avg_duration_per_task_ms": 304751.5525, "avg_output_tokens_per_task": 8046.7025, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.6877776666666667, "accuracy_std_dev": 0.010304212256267955, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11416634666666667, "avg_duration_per_task_ms": 162037.17333333334, "avg_output_tokens_per_task": 8889.293333333333, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6876191428571429, "accuracy_std_dev": 0.0358884483600765, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.03518511571428571, "avg_duration_per_task_ms": 165038.81714285715, "avg_output_tokens_per_task": 10124.521428571428, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6872909230769231, "accuracy_std_dev": 0.021147758508094125, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.006207953177257525, "avg_duration_per_task_ms": 252164.59866220737, "avg_output_tokens_per_task": 9519.789297658863, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02072933, "avg_duration_per_task_ms": 450665.84, "avg_output_tokens_per_task": 10600.05, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6866664285714287, "accuracy_std_dev": 0.041709532236501465, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02076982, "avg_duration_per_task_ms": 414109.71142857143, "avg_output_tokens_per_task": 20509.702857142856, "last_run_timestamp": "2026-10-03T16:11:33.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.6833334999999999, "accuracy_std_dev": 0.07322715191682658, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11095547600000001, "avg_duration_per_task_ms": 69234.286, "avg_output_tokens_per_task": 8865.102, "last_run_timestamp": "2026-10-03T16:27:27.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.07779388246014092, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00909372, "avg_duration_per_task_ms": 253755.2, "avg_output_tokens_per_task": 15941.146666666667, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.68333325, "accuracy_std_dev": 0.06368309125024867, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.05480554, "avg_duration_per_task_ms": 1657352.435, "avg_output_tokens_per_task": 54132.445, "last_run_timestamp": "2026-10-03T08:39:18.140Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.6824557368421051, "accuracy_std_dev": 0.05773499999999998, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.19861939999999997, "avg_duration_per_task_ms": 328265.04210526316, "avg_output_tokens_per_task": 23629.65263157895, "last_run_timestamp": "2026-10-03T16:20:06.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.058584163368230795, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02942621, "avg_duration_per_task_ms": 175208.89333333334, "avg_output_tokens_per_task": 10644.23, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6820807398843931, "accuracy_std_dev": 0.018427811270036426, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.015085965317919076, "avg_duration_per_task_ms": 243924.5809248555, "avg_output_tokens_per_task": 11146.326589595375, "last_run_timestamp": "2026-08-30T00:01:09.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6819047142857143, "accuracy_std_dev": 0.025376904886779485, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007673354285714286, "avg_duration_per_task_ms": 259027.58, "avg_output_tokens_per_task": 11768.4, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6816665000000001, "accuracy_std_dev": 0.02327362599059285, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013877145, "avg_duration_per_task_ms": 70751.9, "avg_output_tokens_per_task": 5304.49, "last_run_timestamp": "2026-10-01T19:04:52.652Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.68, "accuracy_std_dev": 0.01440159456449179, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015966806666666666, "avg_duration_per_task_ms": 259752.2, "avg_output_tokens_per_task": 10038.846666666666, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01554348, "avg_duration_per_task_ms": 297033.57, "avg_output_tokens_per_task": 18106.79, "last_run_timestamp": "2026-10-09T05:05:31.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04451592, "avg_duration_per_task_ms": 466424.76, "avg_output_tokens_per_task": 10206.78, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6761905714285714, "accuracy_std_dev": 0.01880810362474601, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009157782857142858, "avg_duration_per_task_ms": 249382.73428571428, "avg_output_tokens_per_task": 10084.637142857142, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64995528, "avg_duration_per_task_ms": 85231.23, "avg_output_tokens_per_task": 3880.06, "last_run_timestamp": "2026-10-01T08:05:26.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6733333636363636, "accuracy_std_dev": 0.06381806526834732, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06596764181818182, "avg_duration_per_task_ms": 779394.0927272728, "avg_output_tokens_per_task": 18514.336363636365, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.02372680934208296, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11501549333333332, "avg_duration_per_task_ms": 145439.36666666667, "avg_output_tokens_per_task": 56059.84, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.05192585882668568, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.22792725999999996, "avg_duration_per_task_ms": 692249.9666666667, "avg_output_tokens_per_task": 10865.973333333333, "last_run_timestamp": "2026-10-03T16:12:13.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6733330000000001, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00884946, "avg_duration_per_task_ms": 853445.82, "avg_output_tokens_per_task": 18061.82, "last_run_timestamp": "2026-10-09T04:22:40.083Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09000018000000001, "avg_duration_per_task_ms": 1674604.48, "avg_output_tokens_per_task": 17624.46, "last_run_timestamp": "2026-07-18T16:00:08.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.672, "accuracy_std_dev": 0.04759099821604921, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.018393836, "avg_duration_per_task_ms": 255084.928, "avg_output_tokens_per_task": 8922.8, "last_run_timestamp": "2026-08-29T16:33:39.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.672, "accuracy_std_dev": 0.038157690357253, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.01826553, "avg_duration_per_task_ms": 344635.034, "avg_output_tokens_per_task": 19832.934, "last_run_timestamp": "2026-10-07T19:11:59.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.671111, "accuracy_std_dev": 0.034282673881196074, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1170691888888889, "avg_duration_per_task_ms": 122970.47333333333, "avg_output_tokens_per_task": 12647.033333333333, "last_run_timestamp": "2026-10-03T08:12:32.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.67083325, "accuracy_std_dev": 0.019346895388860186, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.017078424999999998, "avg_duration_per_task_ms": 126227.985, "avg_output_tokens_per_task": 21265.575, "last_run_timestamp": "2026-10-03T08:15:42.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6700225369127517, "accuracy_std_dev": 0.06602622409757607, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.02446618791946309, "avg_duration_per_task_ms": 477944.1677852349, "avg_output_tokens_per_task": 16475.68120805369, "last_run_timestamp": "2026-10-03T16:16:54.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.67, "accuracy_std_dev": 0.08041555612411072, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07839768750000001, "avg_duration_per_task_ms": 277010.8, "avg_output_tokens_per_task": 25559.2775, "last_run_timestamp": "2026-10-03T16:03:45.631Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6692307307692308, "accuracy_std_dev": 0.031054260799641166, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.12451571153846153, "avg_duration_per_task_ms": 119043.02846153846, "avg_output_tokens_per_task": 12553.901538461538, "last_run_timestamp": "2026-10-03T08:32:52.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.669204852791878, "accuracy_std_dev": 0.0421484109961455, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.023811266497461927, "avg_duration_per_task_ms": 266573.3654822335, "avg_output_tokens_per_task": 17577.88578680203, "last_run_timestamp": "2026-10-03T16:10:49.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.6685714285714286, "accuracy_std_dev": 0.01875964453559928, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.029877088571428574, "avg_duration_per_task_ms": 242438.32571428572, "avg_output_tokens_per_task": 11057.357142857143, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.668333375, "accuracy_std_dev": 0.05299363562480287, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02315143, "avg_duration_per_task_ms": 206155.715, "avg_output_tokens_per_task": 9305.305, "last_run_timestamp": "2026-10-03T16:13:50.327Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.668, "accuracy_std_dev": 0.03357230340027326, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021775528000000002, "avg_duration_per_task_ms": 221637.98, "avg_output_tokens_per_task": 29545.616, "last_run_timestamp": "2026-10-08T22:26:21.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6658330000000001, "accuracy_std_dev": 0.014790199457749011, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02195716, "avg_duration_per_task_ms": 162018.975, "avg_output_tokens_per_task": 16412.775, "last_run_timestamp": "2026-10-03T16:06:47.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.664762, "accuracy_std_dev": 0.09453067457406919, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08868562, "avg_duration_per_task_ms": 89242.03714285714, "avg_output_tokens_per_task": 8290.754285714285, "last_run_timestamp": "2026-10-03T16:14:25.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.6644296040268456, "accuracy_std_dev": 0.09950169970519207, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.07663720134228189, "avg_duration_per_task_ms": 896985.9765100671, "avg_output_tokens_per_task": 47829.18120805369, "last_run_timestamp": "2026-10-03T16:13:53.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6603704444444444, "accuracy_std_dev": 0.02063470181843689, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11393844222222221, "avg_duration_per_task_ms": 124032.99777777778, "avg_output_tokens_per_task": 12195, "last_run_timestamp": "2026-10-03T08:07:11.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6600001428571428, "accuracy_std_dev": 0.024688651721727035, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02290525142857143, "avg_duration_per_task_ms": 311367.0028571429, "avg_output_tokens_per_task": 10406.782857142856, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00861218, "avg_duration_per_task_ms": 379136.4, "avg_output_tokens_per_task": 24201, "last_run_timestamp": "2026-07-18T16:00:10.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.659524, "accuracy_std_dev": 0.0937819106087553, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.058310568571428564, "avg_duration_per_task_ms": 524817.9057142857, "avg_output_tokens_per_task": 17039.157142857144, "last_run_timestamp": "2026-10-03T16:12:24.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.6588888333333334, "accuracy_std_dev": 0.02354406690595791, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022377286666666666, "avg_duration_per_task_ms": 254720.16666666666, "avg_output_tokens_per_task": 9788.826666666666, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6580951428571429, "accuracy_std_dev": 0.018418057788319512, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.026633637142857144, "avg_duration_per_task_ms": 383567.4857142857, "avg_output_tokens_per_task": 10552.454285714286, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6560493333333334, "accuracy_std_dev": 0.029495577726286477, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.12169699629629627, "avg_duration_per_task_ms": 120224.82148148149, "avg_output_tokens_per_task": 12512.395555555555, "last_run_timestamp": "2026-10-03T08:13:54.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6557788844221106, "accuracy_std_dev": 0.01812503680513505, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.017334296482412057, "avg_duration_per_task_ms": 354124.29145728645, "avg_output_tokens_per_task": 22535.824120603014, "last_run_timestamp": "2026-10-03T08:21:59.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6555554666666666, "accuracy_std_dev": 0.010407070153866542, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.014745425641025642, "avg_duration_per_task_ms": 278015.8512820513, "avg_output_tokens_per_task": 20205.384615384617, "last_run_timestamp": "2026-10-03T08:06:43.564Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.023726809342082957, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03210012, "avg_duration_per_task_ms": 143919.89333333334, "avg_output_tokens_per_task": 17353.273333333334, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.65166675, "accuracy_std_dev": 0.012801703448662586, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.031589305, "avg_duration_per_task_ms": 139715.76, "avg_output_tokens_per_task": 17049.6, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6495237142857143, "accuracy_std_dev": 0.031240416581794835, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021353940000000002, "avg_duration_per_task_ms": 237904.40285714285, "avg_output_tokens_per_task": 10393.82, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6481482222222222, "accuracy_std_dev": 0.02470134810436142, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11660464888888887, "avg_duration_per_task_ms": 145744.04888888888, "avg_output_tokens_per_task": 12598.522222222222, "last_run_timestamp": "2026-10-03T08:13:37.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.6481482020202021, "accuracy_std_dev": 0.0054059166891471805, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.009114631313131312, "avg_duration_per_task_ms": 604721.6666666666, "avg_output_tokens_per_task": 19746.29292929293, "last_run_timestamp": "2026-08-22T16:10:30.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6477778333333334, "accuracy_std_dev": 0.03430091421724631, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09916395333333333, "avg_duration_per_task_ms": 153090.92333333334, "avg_output_tokens_per_task": 12859.136666666667, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6476190000000001, "accuracy_std_dev": 0.02121044324316276, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02434872285714286, "avg_duration_per_task_ms": 154447.84, "avg_output_tokens_per_task": 10538.351428571428, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.647474909090909, "accuracy_std_dev": 0.02520056863029086, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.011397434343434345, "avg_duration_per_task_ms": 317877.1191919192, "avg_output_tokens_per_task": 16792.08282828283, "last_run_timestamp": "2026-10-09T16:52:01.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.6474037386934673, "accuracy_std_dev": 0.11098825656657328, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.08922564070351757, "avg_duration_per_task_ms": 216302.63819095478, "avg_output_tokens_per_task": 23678.577889447235, "last_run_timestamp": "2026-10-03T16:12:24.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6473332, "accuracy_std_dev": 0.027840964224681578, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022036860000000002, "avg_duration_per_task_ms": 301895.372, "avg_output_tokens_per_task": 10556.636, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.646566095477387, "accuracy_std_dev": 0.12986162562313927, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.04628975628140704, "avg_duration_per_task_ms": 2127959.7989949747, "avg_output_tokens_per_task": 71164.3944723618, "last_run_timestamp": "2026-10-03T08:42:41.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.646, "accuracy_std_dev": 0.03453813994991622, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014338291999999999, "avg_duration_per_task_ms": 1162755.396, "avg_output_tokens_per_task": 23357.7, "last_run_timestamp": "2026-10-05T08:23:35.777Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6454080918367348, "accuracy_std_dev": 0.11180720901616487, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.3985840484693877, "avg_duration_per_task_ms": 2129131.2704081633, "avg_output_tokens_per_task": 67490.7780612245, "last_run_timestamp": "2026-10-05T20:21:41.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6433335, "accuracy_std_dev": 0.014272415770172448, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031060726666666667, "avg_duration_per_task_ms": 229987.72666666665, "avg_output_tokens_per_task": 16358.803333333333, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6433333333333333, "accuracy_std_dev": 0.14455548467015086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10531282, "avg_duration_per_task_ms": 1453976.54, "avg_output_tokens_per_task": 18878.353333333333, "last_run_timestamp": "2026-08-08T08:21:31.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.642083375, "accuracy_std_dev": 0.03004338703000005, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.033152745, "avg_duration_per_task_ms": 82992.1225, "avg_output_tokens_per_task": 19582.065, "last_run_timestamp": "2026-10-03T16:20:43.625Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6415409547738694, "accuracy_std_dev": 0.019239053075710364, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.012467824120603016, "avg_duration_per_task_ms": 963027.175879397, "avg_output_tokens_per_task": 22096.447236180906, "last_run_timestamp": "2026-10-03T16:01:59.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.6390477142857143, "accuracy_std_dev": 0.02969234538065464, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.018472702857142857, "avg_duration_per_task_ms": 329519.3171428571, "avg_output_tokens_per_task": 20296.345714285715, "last_run_timestamp": "2026-10-03T16:05:02.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.6383332500000001, "accuracy_std_dev": 0.028431218171008805, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.216643625, "avg_duration_per_task_ms": 1122861.59, "avg_output_tokens_per_task": 30036.18, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.04228062043326973, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08944130666666666, "avg_duration_per_task_ms": 118157.82666666666, "avg_output_tokens_per_task": 12612.513333333334, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.6346803232323233, "accuracy_std_dev": 0.01547599999999999, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.019095151515151516, "avg_duration_per_task_ms": 172977.9898989899, "avg_output_tokens_per_task": 9245.858585858587, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.6336735020408163, "accuracy_std_dev": 0.10052381662193295, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.02281844081632653, "avg_duration_per_task_ms": 906837.293877551, "avg_output_tokens_per_task": 30461.944897959183, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.632496338912134, "accuracy_std_dev": 0.08745539404199149, "run_count": 5, "total_questions": 239, "avg_cost_per_task_usd": 0.1367811171548117, "avg_duration_per_task_ms": 350756.1715481171, "avg_output_tokens_per_task": 10337.066945606695, "last_run_timestamp": "2026-08-29T16:44:17.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6314284285714286, "accuracy_std_dev": 0.018418242461965675, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.028592208571428572, "avg_duration_per_task_ms": 225611.1542857143, "avg_output_tokens_per_task": 10242.23142857143, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.631111, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02504141333333333, "avg_duration_per_task_ms": 168170.98, "avg_output_tokens_per_task": 11245.873333333333, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6300762106598985, "accuracy_std_dev": 0.03222693627952934, "run_count": 16, "total_questions": 788, "avg_cost_per_task_usd": 0.021151480964467007, "avg_duration_per_task_ms": 298993.4543147208, "avg_output_tokens_per_task": 18695.17766497462, "last_run_timestamp": "2026-10-09T00:49:56.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.63, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07672618, "avg_duration_per_task_ms": 231628.06, "avg_output_tokens_per_task": 10580.58, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.6278538630136987, "accuracy_std_dev": 0.07680767503894964, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.09331428767123287, "avg_duration_per_task_ms": 1104390.4520547944, "avg_output_tokens_per_task": 65652.6198630137, "last_run_timestamp": "2026-10-03T16:14:02.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6266667499999999, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.116286155, "avg_duration_per_task_ms": 669417.115, "avg_output_tokens_per_task": 30726.725, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.6266663333333334, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02545802, "avg_duration_per_task_ms": 175220.38666666666, "avg_output_tokens_per_task": 11691.16, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.6234567925925926, "accuracy_std_dev": 0.06433938027058783, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.022850422222222222, "avg_duration_per_task_ms": 254929.7462962963, "avg_output_tokens_per_task": 24515.818518518518, "last_run_timestamp": "2026-10-03T16:12:24.693Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6216667499999999, "accuracy_std_dev": 0.04997217837144482, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08547625, "avg_duration_per_task_ms": 106037.93, "avg_output_tokens_per_task": 11996.815, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.6211111666666667, "accuracy_std_dev": 0.18460855717654465, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13787135666666667, "avg_duration_per_task_ms": 390996.2266666667, "avg_output_tokens_per_task": 8010.466666666666, "last_run_timestamp": "2026-10-03T16:12:26.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6211110000000001, "accuracy_std_dev": 0.041171085383636265, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09511168999999998, "avg_duration_per_task_ms": 55650.66, "avg_output_tokens_per_task": 3586.0333333333333, "last_run_timestamp": "2026-10-03T16:14:18.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.6204761428571429, "accuracy_std_dev": 0.025224357448582882, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06417251142857144, "avg_duration_per_task_ms": 29559.602857142858, "avg_output_tokens_per_task": 20035.14285714286, "last_run_timestamp": "2026-10-03T16:18:39.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6180555, "accuracy_std_dev": 0.00694450000000002, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.025814302083333334, "avg_duration_per_task_ms": 76875.0625, "avg_output_tokens_per_task": 17301.635416666668, "last_run_timestamp": "2026-08-29T16:33:38.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6174497919463087, "accuracy_std_dev": 0.011290615681716505, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.11355312416107383, "avg_duration_per_task_ms": 671627.1577181208, "avg_output_tokens_per_task": 32151.05033557047, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6158248141414141, "accuracy_std_dev": 0.2285695922821756, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.020359361616161618, "avg_duration_per_task_ms": 340617.86262626265, "avg_output_tokens_per_task": 10409.925252525252, "last_run_timestamp": "2026-09-26T08:14:40.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.6155553333333333, "accuracy_std_dev": 0.04228062043326968, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09065885333333334, "avg_duration_per_task_ms": 148176.83333333334, "avg_output_tokens_per_task": 3889.28, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.613333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01616616, "avg_duration_per_task_ms": 108573.58, "avg_output_tokens_per_task": 21305.1, "last_run_timestamp": "2026-10-03T16:13:54.745Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.613333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07264287999999999, "avg_duration_per_task_ms": 1482714.84, "avg_output_tokens_per_task": 23767.86, "last_run_timestamp": "2026-07-18T16:00:08.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.6114287142857142, "accuracy_std_dev": 0.0649995778815179, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.037125854285714285, "avg_duration_per_task_ms": 389444.8, "avg_output_tokens_per_task": 19927.58, "last_run_timestamp": "2026-10-03T16:12:10.894Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.608, "accuracy_std_dev": 0.11488351975457575, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.058990216, "avg_duration_per_task_ms": 582295.388, "avg_output_tokens_per_task": 35784.112, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6076188571428572, "accuracy_std_dev": 0.03274303997331158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.046068719999999994, "avg_duration_per_task_ms": 191059.44, "avg_output_tokens_per_task": 3405.154285714286, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6060269955357143, "accuracy_std_dev": 0.032348691330028045, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.19772456473214287, "avg_duration_per_task_ms": 188670.52455357142, "avg_output_tokens_per_task": 13047.183035714286, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.605442137755102, "accuracy_std_dev": 0.02995845369815299, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.041257920918367344, "avg_duration_per_task_ms": 86489.80612244898, "avg_output_tokens_per_task": 24455.91581632653, "last_run_timestamp": "2026-10-03T16:12:29.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.6033332222222223, "accuracy_std_dev": 0.027397264890973577, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04748484000000001, "avg_duration_per_task_ms": 91610.96, "avg_output_tokens_per_task": 4621.971111111111, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.603333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.030717460000000002, "avg_duration_per_task_ms": 422658.18, "avg_output_tokens_per_task": 11680.98, "last_run_timestamp": "2026-07-18T16:00:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5979899899497488, "accuracy_std_dev": 0.022095125194471644, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.020116060301507536, "avg_duration_per_task_ms": 100330.36180904522, "avg_output_tokens_per_task": 4353.20351758794, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.5933333333333334, "accuracy_std_dev": 0.02880349773127486, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21649743333333332, "avg_duration_per_task_ms": 191120.76666666666, "avg_output_tokens_per_task": 14122.313333333334, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.5913978951612903, "accuracy_std_dev": 0.012203964545998971, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.156355625, "avg_duration_per_task_ms": 554104.3709677419, "avg_output_tokens_per_task": 27726.576612903227, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5881481111111111, "accuracy_std_dev": 0.007554077566224961, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029889999999999996, "avg_duration_per_task_ms": 78610.16, "avg_output_tokens_per_task": 8108.122222222222, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.579037868556701, "accuracy_std_dev": 0.04024006578337068, "run_count": 8, "total_questions": 388, "avg_cost_per_task_usd": 0.07504983247422681, "avg_duration_per_task_ms": 90151.55927835051, "avg_output_tokens_per_task": 5664.682989690722, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.572072, "accuracy_std_dev": 0.02796372233607127, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.09883952702702703, "avg_duration_per_task_ms": 105307.2027027027, "avg_output_tokens_per_task": 3660.560810810811, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.571111, "accuracy_std_dev": 0.008314764498569157, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028662713333333336, "avg_duration_per_task_ms": 78035.85333333333, "avg_output_tokens_per_task": 8001.593333333333, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5693332, "accuracy_std_dev": 0.03832036650346651, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0105344, "avg_duration_per_task_ms": 254767.968, "avg_output_tokens_per_task": 16832.444, "last_run_timestamp": "2026-08-29T16:33:38.124Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5641890743243243, "accuracy_std_dev": 0.04318929456634928, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.18690275337837842, "avg_duration_per_task_ms": 1880221.320945946, "avg_output_tokens_per_task": 41218.97635135135, "last_run_timestamp": "2026-10-03T16:17:09.625Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.5622223333333333, "accuracy_std_dev": 0.03456997912768952, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.23936231333333335, "avg_duration_per_task_ms": 165980.19333333333, "avg_output_tokens_per_task": 13303.673333333334, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.5574712183908046, "accuracy_std_dev": 0.07593309995634047, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.05341043390804598, "avg_duration_per_task_ms": 469408.1063218391, "avg_output_tokens_per_task": 9846.341954022988, "last_run_timestamp": "2026-10-04T00:00:28.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.5572390808080808, "accuracy_std_dev": 0.08974967405616299, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04370982828282828, "avg_duration_per_task_ms": 3081206.1666666665, "avg_output_tokens_per_task": 93737.79292929293, "last_run_timestamp": "2026-10-03T16:33:53.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5521540748299321, "accuracy_std_dev": 0.03338584001572458, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.0026003809523809525, "avg_duration_per_task_ms": 390785.49659863947, "avg_output_tokens_per_task": 6412.292517006803, "last_run_timestamp": "2026-10-03T16:14:16.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5511113333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.035753513333333334, "avg_duration_per_task_ms": 61316.40666666667, "avg_output_tokens_per_task": 6659.566666666667, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.54833325, "accuracy_std_dev": 0.09993065790305547, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0400913925, "avg_duration_per_task_ms": 97587.1625, "avg_output_tokens_per_task": 4289.61, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5416667142857143, "accuracy_std_dev": 0.10219251042847255, "run_count": 7, "total_questions": 336, "avg_cost_per_task_usd": 0.06745207738095238, "avg_duration_per_task_ms": 446328.3333333333, "avg_output_tokens_per_task": 28269.869047619046, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.5409524285714286, "accuracy_std_dev": 0.098072099444173, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.027534851428571427, "avg_duration_per_task_ms": 272348.91142857145, "avg_output_tokens_per_task": 7453.248571428571, "last_run_timestamp": "2026-10-03T16:13:14.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5351473333333333, "accuracy_std_dev": 0.016949235571913614, "run_count": 6, "total_questions": 294, "avg_cost_per_task_usd": 0.025283034013605446, "avg_duration_per_task_ms": 1563224.0068027212, "avg_output_tokens_per_task": 123876.9387755102, "last_run_timestamp": "2026-10-03T16:10:46.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5333335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10219375, "avg_duration_per_task_ms": 375443.12, "avg_output_tokens_per_task": 17495.06, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.526667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02328844, "avg_duration_per_task_ms": 1276910.42, "avg_output_tokens_per_task": 153904.82, "last_run_timestamp": "2026-07-18T16:00:10.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.526667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03416076, "avg_duration_per_task_ms": 1845834.04, "avg_output_tokens_per_task": 105339.92, "last_run_timestamp": "2026-07-18T16:00:08.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.523035243902439, "accuracy_std_dev": 0.26305710961614404, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.01812669918699187, "avg_duration_per_task_ms": 110026.72764227643, "avg_output_tokens_per_task": 18887.439024390245, "last_run_timestamp": "2026-08-29T17:06:47.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5221089591836734, "accuracy_std_dev": 0.023889008586586408, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.13644156632653062, "avg_duration_per_task_ms": 73125.92346938775, "avg_output_tokens_per_task": 2553.7397959183672, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5201006834170854, "accuracy_std_dev": 0.01697055860112964, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.06560986683417085, "avg_duration_per_task_ms": 298265.93216080405, "avg_output_tokens_per_task": 4016.8693467336684, "last_run_timestamp": "2026-10-04T00:00:34.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517500125, "accuracy_std_dev": 0.024930426803594342, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0282700725, "avg_duration_per_task_ms": 444815, "avg_output_tokens_per_task": 58343.125, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5156599127516779, "accuracy_std_dev": 0.03449671802746843, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.024214489932885906, "avg_duration_per_task_ms": 190286.47651006712, "avg_output_tokens_per_task": 48937.10738255033, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.5104164999999999, "accuracy_std_dev": 0.003472500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.024820281250000003, "avg_duration_per_task_ms": 480070.4583333333, "avg_output_tokens_per_task": 9578.8125, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5083332500000001, "accuracy_std_dev": 0.026404802805313633, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12635733, "avg_duration_per_task_ms": 53878.245, "avg_output_tokens_per_task": 2800.47, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.5045660136986302, "accuracy_std_dev": 0.02732875245760204, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.05861860273972602, "avg_duration_per_task_ms": 2642554.2671232875, "avg_output_tokens_per_task": 40736.36301369863, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5033670303030303, "accuracy_std_dev": 0.030340499999999992, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.01769851515151515, "avg_duration_per_task_ms": 135563.53535353535, "avg_output_tokens_per_task": 30988.575757575756, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.5033669747474748, "accuracy_std_dev": 0.027292446564651823, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.025276146464646466, "avg_duration_per_task_ms": 577739.3712121212, "avg_output_tokens_per_task": 53598.469696969696, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5013551951219513, "accuracy_std_dev": 0.01245431828082131, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.020900845528455284, "avg_duration_per_task_ms": 1272391.512195122, "avg_output_tokens_per_task": 112484.65853658537, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.5011262027027026, "accuracy_std_dev": 0.034811519447191935, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.15286014527027028, "avg_duration_per_task_ms": 1793120.7567567567, "avg_output_tokens_per_task": 35316.18243243243, "last_run_timestamp": "2026-10-03T16:05:51.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.49958325000000003, "accuracy_std_dev": 0.03216451093888264, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.026658079999999997, "avg_duration_per_task_ms": 280333.52, "avg_output_tokens_per_task": 19065.1925, "last_run_timestamp": "2026-10-03T16:15:51.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23109727999999996, "avg_duration_per_task_ms": 80168.37, "avg_output_tokens_per_task": 2746.03, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.489899005050505, "accuracy_std_dev": 0.009452416119701875, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.014275666666666666, "avg_duration_per_task_ms": 117584.4797979798, "avg_output_tokens_per_task": 3141.5757575757575, "last_run_timestamp": "2026-10-03T16:05:51.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.48888899999999996, "accuracy_std_dev": 0.01872462563043652, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06696013, "avg_duration_per_task_ms": 315550.43333333335, "avg_output_tokens_per_task": 4089.9266666666667, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48833333333333334, "accuracy_std_dev": 0.018831396165859726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03323320666666667, "avg_duration_per_task_ms": 665027.6466666666, "avg_output_tokens_per_task": 61694.416666666664, "last_run_timestamp": "2026-10-03T16:04:32.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.4852150080645161, "accuracy_std_dev": 0.023420064714684302, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.028619552419354842, "avg_duration_per_task_ms": 386610.2439516129, "avg_output_tokens_per_task": 66355.3125, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.4842343648648649, "accuracy_std_dev": 0.02401659838297026, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.023849905405405405, "avg_duration_per_task_ms": 836568.3851351351, "avg_output_tokens_per_task": 22266.58108108108, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.48, "accuracy_std_dev": 0.0241687453956551, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034828557142857146, "avg_duration_per_task_ms": 43808.188571428575, "avg_output_tokens_per_task": 3663.8714285714286, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.4777776666666667, "accuracy_std_dev": 0.013698571612479242, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12510532, "avg_duration_per_task_ms": 53067.566666666666, "avg_output_tokens_per_task": 2709.2933333333335, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.4776285234899329, "accuracy_std_dev": 0.010657414972163225, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.1647015033557047, "avg_duration_per_task_ms": 2471417.852348993, "avg_output_tokens_per_task": 44160.27181208054, "last_run_timestamp": "2026-10-03T16:12:22.579Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.47583349999999996, "accuracy_std_dev": 0.03650536870790377, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02416941, "avg_duration_per_task_ms": 59754.7125, "avg_output_tokens_per_task": 3775.0125, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.47333325000000004, "accuracy_std_dev": 0.025603927651583064, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0251383825, "avg_duration_per_task_ms": 126301.635, "avg_output_tokens_per_task": 3791.835, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03429832, "avg_duration_per_task_ms": 458175.58, "avg_output_tokens_per_task": 37835.14, "last_run_timestamp": "2026-07-18T16:00:08.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.4727888571428572, "accuracy_std_dev": 0.053216013468691904, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.026510076530612242, "avg_duration_per_task_ms": 2225913.1275510206, "avg_output_tokens_per_task": 135590.98469387754, "last_run_timestamp": "2026-10-03T16:09:05.687Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.472222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.02306027083333333, "avg_duration_per_task_ms": 25624.729166666668, "avg_output_tokens_per_task": 1174.2916666666667, "last_run_timestamp": "2026-07-18T16:00:08.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.4721422449567723, "accuracy_std_dev": 0.09390832636273526, "run_count": 14, "total_questions": 694, "avg_cost_per_task_usd": 0.04128501008645533, "avg_duration_per_task_ms": 719831.6239193083, "avg_output_tokens_per_task": 196500.71037463978, "last_run_timestamp": "2026-10-03T16:19:24.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.47047614285714284, "accuracy_std_dev": 0.020426164806293848, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021544591428571434, "avg_duration_per_task_ms": 680725.9685714286, "avg_output_tokens_per_task": 21489.04857142857, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.46952399999999994, "accuracy_std_dev": 0.034523096873170074, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.018877348571428573, "avg_duration_per_task_ms": 90973.66571428571, "avg_output_tokens_per_task": 3606.62, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.46833325000000003, "accuracy_std_dev": 0.03295620410389368, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09710940000000001, "avg_duration_per_task_ms": 99117.8225, "avg_output_tokens_per_task": 2749.945, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4682815259515572, "accuracy_std_dev": 0.039807100596362034, "run_count": 6, "total_questions": 289, "avg_cost_per_task_usd": 0.04939322491349481, "avg_duration_per_task_ms": 229004.3321799308, "avg_output_tokens_per_task": 98239.9169550173, "last_run_timestamp": "2026-10-03T16:14:31.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.4662164729729729, "accuracy_std_dev": 0.03892660093903227, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04219703378378378, "avg_duration_per_task_ms": 711257.6756756756, "avg_output_tokens_per_task": 98782.22297297297, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.465926, "accuracy_std_dev": 0.03230940398363575, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.022285648888888884, "avg_duration_per_task_ms": 222200.71555555557, "avg_output_tokens_per_task": 3598.428888888889, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4608501677852349, "accuracy_std_dev": 0.034994868451198195, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04671855033557047, "avg_duration_per_task_ms": 767600.1241610738, "avg_output_tokens_per_task": 101728.0033557047, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.46068351282051284, "accuracy_std_dev": 0.04225414673955088, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.01900871794871795, "avg_duration_per_task_ms": 170098.1794871795, "avg_output_tokens_per_task": 35854.89743589744, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.45958325, "accuracy_std_dev": 0.040771748628032865, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.052828784999999996, "avg_duration_per_task_ms": 158699.07, "avg_output_tokens_per_task": 3758.8575, "last_run_timestamp": "2026-10-03T16:12:33.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4577777777777778, "accuracy_std_dev": 0.04455537267959145, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05422914222222222, "avg_duration_per_task_ms": 126062.82666666666, "avg_output_tokens_per_task": 3580.0733333333333, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.4570707575757576, "accuracy_std_dev": 0.044267226719521297, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.05378886363636363, "avg_duration_per_task_ms": 73384.54040404041, "avg_output_tokens_per_task": 3408.1742424242425, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.4560182083333333, "accuracy_std_dev": 0.022912149746368182, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.08057918055555555, "avg_duration_per_task_ms": 52530.756944444445, "avg_output_tokens_per_task": 3473.4930555555557, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4511785656565656, "accuracy_std_dev": 0.03515039216708685, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04521708585858586, "avg_duration_per_task_ms": 779859.6717171717, "avg_output_tokens_per_task": 105433.07575757576, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.45095228571428575, "accuracy_std_dev": 0.02741314960126298, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.020993765714285715, "avg_duration_per_task_ms": 566234.7285714286, "avg_output_tokens_per_task": 21154.594285714287, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.45, "accuracy_std_dev": 0.01784692109767581, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03456686, "avg_duration_per_task_ms": 476526.7733333333, "avg_output_tokens_per_task": 23360.6, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.44750012499999997, "accuracy_std_dev": 0.06501608615073176, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.16146158749999998, "avg_duration_per_task_ms": 120680.0125, "avg_output_tokens_per_task": 3281.1225, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.445416625, "accuracy_std_dev": 0.03036532482099237, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0408187325, "avg_duration_per_task_ms": 287335.82, "avg_output_tokens_per_task": 3635.3075, "last_run_timestamp": "2026-10-03T16:17:30.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.4444443333333334, "accuracy_std_dev": 0.030952277087298266, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03472188, "avg_duration_per_task_ms": 283560.81333333335, "avg_output_tokens_per_task": 21592.06, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4441665, "accuracy_std_dev": 0.023555566693459103, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03439918, "avg_duration_per_task_ms": 113438.9075, "avg_output_tokens_per_task": 3146.3175, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.44333325, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025700495, "avg_duration_per_task_ms": 61023.305, "avg_output_tokens_per_task": 2441.375, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.44186027325581395, "accuracy_std_dev": 0.0336023614676719, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.05302273255813953, "avg_duration_per_task_ms": 228343.90697674418, "avg_output_tokens_per_task": 2116.357558139535, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4416665, "accuracy_std_dev": 0.017873086533948188, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.104832345, "avg_duration_per_task_ms": 890298.59, "avg_output_tokens_per_task": 21334.155, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2288897, "avg_duration_per_task_ms": 83836.68, "avg_output_tokens_per_task": 2734.56, "last_run_timestamp": "2026-07-18T08:04:08.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.43869733333333333, "accuracy_std_dev": 0.023050038409813497, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.01657819540229885, "avg_duration_per_task_ms": 200645.3132183908, "avg_output_tokens_per_task": 2785.6264367816093, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4383965367088608, "accuracy_std_dev": 0.03960836597472182, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.05240046329113924, "avg_duration_per_task_ms": 422067.2658227848, "avg_output_tokens_per_task": 20533.427848101266, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.43777799999999994, "accuracy_std_dev": 0.03861798340151904, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.19450383999999998, "avg_duration_per_task_ms": 50798.87333333334, "avg_output_tokens_per_task": 2047.5333333333333, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.011330936570096718, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02595394666666667, "avg_duration_per_task_ms": 66273.3, "avg_output_tokens_per_task": 2407.2933333333335, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21387673999999998, "avg_duration_per_task_ms": 47267.2, "avg_output_tokens_per_task": 2224.56, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.4372223333333333, "accuracy_std_dev": 0.025270150254576823, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.027693089999999997, "avg_duration_per_task_ms": 78315.11, "avg_output_tokens_per_task": 2793.7166666666667, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.4362416308724832, "accuracy_std_dev": 0.02031037140436711, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.0464321610738255, "avg_duration_per_task_ms": 136873.4798657718, "avg_output_tokens_per_task": 2090.986577181208, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.435, "accuracy_std_dev": 0.020749805890658335, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.074401825, "avg_duration_per_task_ms": 778012.545, "avg_output_tokens_per_task": 20313.29, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.43434354545454545, "accuracy_std_dev": 0.022037645087440712, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.06906736868686869, "avg_duration_per_task_ms": 323353.22222222225, "avg_output_tokens_per_task": 3378.3585858585857, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.43416675, "accuracy_std_dev": 0.04222137919570487, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.400907955, "avg_duration_per_task_ms": 107994.6975, "avg_output_tokens_per_task": 3151.7075, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43416662500000003, "accuracy_std_dev": 0.01613729505630899, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2170375225, "avg_duration_per_task_ms": 46329.7025, "avg_output_tokens_per_task": 2265.4175, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4337431393442623, "accuracy_std_dev": 0.021677717946315293, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.032824139344262294, "avg_duration_per_task_ms": 478154.40573770495, "avg_output_tokens_per_task": 4694.745901639344, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.431111, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029769013333333334, "avg_duration_per_task_ms": 630152.1266666667, "avg_output_tokens_per_task": 18760.18666666667, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.43089426016260163, "accuracy_std_dev": 0.03483555910043644, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.04202976422764227, "avg_duration_per_task_ms": 58240.52032520325, "avg_output_tokens_per_task": 2134.682926829268, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.43083325, "accuracy_std_dev": 0.031213896924086562, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029063365, "avg_duration_per_task_ms": 761962.965, "avg_output_tokens_per_task": 19001.86, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.24811575000000002, "avg_duration_per_task_ms": 46287.18, "avg_output_tokens_per_task": 2565.8, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.42787744843049325, "accuracy_std_dev": 0.02999847077995481, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.024826468609865468, "avg_duration_per_task_ms": 337438.9708520179, "avg_output_tokens_per_task": 3556.2600896860986, "last_run_timestamp": "2026-10-09T00:15:21.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.42666699999999996, "accuracy_std_dev": 0.04000000000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.031438360000000005, "avg_duration_per_task_ms": 468393.89, "avg_output_tokens_per_task": 22584.78, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.425833375, "accuracy_std_dev": 0.013616710018737092, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.024625232500000004, "avg_duration_per_task_ms": 228841.615, "avg_output_tokens_per_task": 3561.265, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.4247621428571428, "accuracy_std_dev": 0.04058525139022997, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05475465714285714, "avg_duration_per_task_ms": 141536.79714285713, "avg_output_tokens_per_task": 3538.397142857143, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.42144822950819677, "accuracy_std_dev": 0.025118027628776907, "run_count": 10, "total_questions": 488, "avg_cost_per_task_usd": 0.050889221311475406, "avg_duration_per_task_ms": 182960.04508196723, "avg_output_tokens_per_task": 4075.3032786885246, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4166667045454545, "accuracy_std_dev": 0.029461897667544087, "run_count": 9, "total_questions": 440, "avg_cost_per_task_usd": 0.011195077272727273, "avg_duration_per_task_ms": 112204.60909090909, "avg_output_tokens_per_task": 3271.315909090909, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.4166667027027027, "accuracy_std_dev": 0.06529892179218752, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.23566864189189185, "avg_duration_per_task_ms": 251017.3918918919, "avg_output_tokens_per_task": 9861.692567567568, "last_run_timestamp": "2026-10-04T00:00:34.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.4166666, "accuracy_std_dev": 0.024435281744590136, "run_count": 8, "total_questions": 390, "avg_cost_per_task_usd": 0.03287333333333334, "avg_duration_per_task_ms": 742207.0153846154, "avg_output_tokens_per_task": 7710.807692307692, "last_run_timestamp": "2026-10-09T01:20:30.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.4125, "accuracy_std_dev": 0.12760346564455058, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.023525742499999995, "avg_duration_per_task_ms": 178134.34, "avg_output_tokens_per_task": 3278.42, "last_run_timestamp": "2026-10-03T16:09:19.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.408045959770115, "accuracy_std_dev": 0.16546342883327816, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.14008307183908045, "avg_duration_per_task_ms": 345147.8563218391, "avg_output_tokens_per_task": 46487.37356321839, "last_run_timestamp": "2026-10-03T16:22:23.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40777783333333334, "accuracy_std_dev": 0.01781244190087991, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025491406666666664, "avg_duration_per_task_ms": 73232.91333333333, "avg_output_tokens_per_task": 2860.7966666666666, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.40541675, "accuracy_std_dev": 0.021209148643627825, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06368001, "avg_duration_per_task_ms": 101217.8175, "avg_output_tokens_per_task": 2945.865, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.4053599698492462, "accuracy_std_dev": 0.008101744842933528, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.06672347236180905, "avg_duration_per_task_ms": 212067.50753768845, "avg_output_tokens_per_task": 8080.718592964824, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.40333325000000003, "accuracy_std_dev": 0.034960136029304874, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05923553749999999, "avg_duration_per_task_ms": 320941.4325, "avg_output_tokens_per_task": 7834.6225, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.402500125, "accuracy_std_dev": 0.02559013346212511, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.026921807500000002, "avg_duration_per_task_ms": 293558.7475, "avg_output_tokens_per_task": 4184.715, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.400833375, "accuracy_std_dev": 0.029332849192234545, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0642667, "avg_duration_per_task_ms": 297648.6875, "avg_output_tokens_per_task": 11410.885, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.40083312500000007, "accuracy_std_dev": 0.02459377320805766, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07675446250000001, "avg_duration_per_task_ms": 122147.305, "avg_output_tokens_per_task": 3115.835, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.398333375, "accuracy_std_dev": 0.05074458509766707, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.048334209999999996, "avg_duration_per_task_ms": 87716.1125, "avg_output_tokens_per_task": 3723.8325, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.39726041780821914, "accuracy_std_dev": 0.26615553243962864, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.011427219178082192, "avg_duration_per_task_ms": 86701.55479452055, "avg_output_tokens_per_task": 2538.931506849315, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.396667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0528661, "avg_duration_per_task_ms": 135118.22, "avg_output_tokens_per_task": 7825.32, "last_run_timestamp": "2026-07-25T08:07:19.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.39521795358649786, "accuracy_std_dev": 0.015940999687598014, "run_count": 5, "total_questions": 237, "avg_cost_per_task_usd": 0.009386248945147679, "avg_duration_per_task_ms": 65821.63713080168, "avg_output_tokens_per_task": 1124.5696202531647, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.3944446666666666, "accuracy_std_dev": 0.015947545189011245, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07351260000000001, "avg_duration_per_task_ms": 297595.3333333333, "avg_output_tokens_per_task": 7950.953333333333, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.39249987500000005, "accuracy_std_dev": 0.016811927398706402, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0369896575, "avg_duration_per_task_ms": 191395.045, "avg_output_tokens_per_task": 3738.2225, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3912555560538117, "accuracy_std_dev": 0.029116472984579173, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.059476125560538114, "avg_duration_per_task_ms": 375172.7399103139, "avg_output_tokens_per_task": 3892.973094170404, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.39, "accuracy_std_dev": 0.014782396941407474, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028387520000000006, "avg_duration_per_task_ms": 78569.82333333333, "avg_output_tokens_per_task": 4258.516666666666, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.389166625, "accuracy_std_dev": 0.02355558880572453, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.025453972499999998, "avg_duration_per_task_ms": 227372.8075, "avg_output_tokens_per_task": 4499.2425, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.38666671428571425, "accuracy_std_dev": 0.008728715610219052, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014610905714285714, "avg_duration_per_task_ms": 90807.17428571428, "avg_output_tokens_per_task": 601.44, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.37924317696160265, "accuracy_std_dev": 0.017662856537252282, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.012139260434056762, "avg_duration_per_task_ms": 36090.01335559265, "avg_output_tokens_per_task": 625.2086811352253, "last_run_timestamp": "2026-10-09T15:32:27.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.37583325, "accuracy_std_dev": 0.03264919388265964, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.051693865, "avg_duration_per_task_ms": 72470.795, "avg_output_tokens_per_task": 4363.9325, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.373333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07157664, "avg_duration_per_task_ms": 16604.76, "avg_output_tokens_per_task": 863.76, "last_run_timestamp": "2026-07-18T16:00:09.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.373333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.030392839999999997, "avg_duration_per_task_ms": 86392.54, "avg_output_tokens_per_task": 5907.74, "last_run_timestamp": "2026-07-18T08:03:14.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.3714284285714286, "accuracy_std_dev": 0.017715366073046326, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03264426571428572, "avg_duration_per_task_ms": 156269.61714285714, "avg_output_tokens_per_task": 4433.125714285714, "last_run_timestamp": "2026-10-03T16:07:02.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.37063948837209304, "accuracy_std_dev": 0.0721313728137764, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.004010601744186046, "avg_duration_per_task_ms": 293679.6831395349, "avg_output_tokens_per_task": 4202.110465116279, "last_run_timestamp": "2026-10-03T16:11:33.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.36476185714285714, "accuracy_std_dev": 0.029753169004943562, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07516260571428572, "avg_duration_per_task_ms": 18915.674285714285, "avg_output_tokens_per_task": 875.96, "last_run_timestamp": "2026-10-03T16:05:34.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.36416687499999995, "accuracy_std_dev": 0.07292984785641181, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0056979475000000016, "avg_duration_per_task_ms": 133944.15, "avg_output_tokens_per_task": 3653.925, "last_run_timestamp": "2026-10-03T16:07:05.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.3567604417670683, "accuracy_std_dev": 0.024567950668299542, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.0795283875502008, "avg_duration_per_task_ms": 211758.7168674699, "avg_output_tokens_per_task": 7341.082329317269, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.3566663333333333, "accuracy_std_dev": 0.016996731711975962, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.043029019999999994, "avg_duration_per_task_ms": 111012.42666666667, "avg_output_tokens_per_task": 4511.693333333334, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35592184615384614, "accuracy_std_dev": 0.02760835600602419, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.07156286996336997, "avg_duration_per_task_ms": 377710.5695970696, "avg_output_tokens_per_task": 7880.789377289378, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.3367119324324324, "accuracy_std_dev": 0.23078482961913535, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.014007192567567568, "avg_duration_per_task_ms": 1521898.9797297297, "avg_output_tokens_per_task": 94037.88851351352, "last_run_timestamp": "2026-10-03T16:12:24.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.33666633333333335, "accuracy_std_dev": 0.004714045207910321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029078933333333334, "avg_duration_per_task_ms": 79568.04666666666, "avg_output_tokens_per_task": 3568.733333333333, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.3358587171717172, "accuracy_std_dev": 0.05280919547472296, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.02850434343434343, "avg_duration_per_task_ms": 105990.88888888889, "avg_output_tokens_per_task": 1237.1767676767677, "last_run_timestamp": "2026-10-03T16:14:36.877Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.3296970909090909, "accuracy_std_dev": 0.023845300316279384, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08440126545454546, "avg_duration_per_task_ms": 151545.54545454544, "avg_output_tokens_per_task": 8175.638181818182, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.3206666, "accuracy_std_dev": 0.3296116389532384, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.019227021999999996, "avg_duration_per_task_ms": 58892.988, "avg_output_tokens_per_task": 10442.698, "last_run_timestamp": "2026-10-03T16:12:07.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.30896692982456136, "accuracy_std_dev": 0.03129504337294144, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.01647059649122807, "avg_duration_per_task_ms": 365014.59064327483, "avg_output_tokens_per_task": 4454.833333333333, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.30416662499999997, "accuracy_std_dev": 0.03799677343846942, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.16497433250000001, "avg_duration_per_task_ms": 125643.485, "avg_output_tokens_per_task": 1850.835, "last_run_timestamp": "2026-10-03T16:16:35.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.2844443333333333, "accuracy_std_dev": 0.031894788459698065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02081186, "avg_duration_per_task_ms": 62184.12666666666, "avg_output_tokens_per_task": 3804.7533333333336, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.28000025, "accuracy_std_dev": 0.027080189561513406, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02535222, "avg_duration_per_task_ms": 67904.845, "avg_output_tokens_per_task": 4023.05, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.2797619910714286, "accuracy_std_dev": 0.1899431271985133, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.15921619642857143, "avg_duration_per_task_ms": 942346.96875, "avg_output_tokens_per_task": 6098.834821428572, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2622223333333333, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08337879999999999, "avg_duration_per_task_ms": 576772.3666666667, "avg_output_tokens_per_task": 5220.566666666667, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.260417, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.027361125, "avg_duration_per_task_ms": 394399.5416666667, "avg_output_tokens_per_task": 4477.166666666667, "last_run_timestamp": "2026-07-18T08:04:09.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02790544, "avg_duration_per_task_ms": 464213.12, "avg_output_tokens_per_task": 5425.66, "last_run_timestamp": "2026-07-18T08:04:25.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24200919178082192, "accuracy_std_dev": 0.027381797668443095, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.07592458219178082, "avg_duration_per_task_ms": 576526.0342465753, "avg_output_tokens_per_task": 5262.068493150685, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.2325, "accuracy_std_dev": 0.06851085072271398, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.013581069999999997, "avg_duration_per_task_ms": 21748.2225, "avg_output_tokens_per_task": 1125.92, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.19666649999999997, "accuracy_std_dev": 0.023804971471312466, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00425299, "avg_duration_per_task_ms": 15962.445, "avg_output_tokens_per_task": 1579.045, "last_run_timestamp": "2026-10-03T16:07:44.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.18791944295302013, "accuracy_std_dev": 0.10708233389741122, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.20449519798657714, "avg_duration_per_task_ms": 608397.9966442953, "avg_output_tokens_per_task": 10862.70134228188, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.18766967073170732, "accuracy_std_dev": 0.018396157170452745, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.011055426829268291, "avg_duration_per_task_ms": 138140.34552845528, "avg_output_tokens_per_task": 5390.59756097561, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.17999985714285713, "accuracy_std_dev": 0.12173356543044388, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.013454491428571429, "avg_duration_per_task_ms": 61907.58571428571, "avg_output_tokens_per_task": 1212.5742857142857, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.13333333333333333, "accuracy_std_dev": 0.021430473285072877, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007961373333333334, "avg_duration_per_task_ms": 80728.30333333333, "avg_output_tokens_per_task": 2469.17, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.11833325, "accuracy_std_dev": 0.018484159885899604, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007158584999999999, "avg_duration_per_task_ms": 55526.67, "avg_output_tokens_per_task": 2353.445, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.108889, "accuracy_std_dev": 0.008314764498569193, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006775626666666668, "avg_duration_per_task_ms": 60620.74, "avg_output_tokens_per_task": 2327.0066666666667, "last_run_timestamp": "2026-10-01T19:04:36.194Z" } ] }