{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-10-18", "retrieved_at": "2026-10-18T01:45:34.625Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2148, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.964646, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09468125757575757, "avg_duration_per_task_ms": 101980.28282828283, "avg_output_tokens_per_task": 8439.414141414141, "last_run_timestamp": "2026-10-14T15:00:03.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9483723333333332, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20306945454545455, "avg_duration_per_task_ms": 134835.03535353535, "avg_output_tokens_per_task": 16798.409090909092, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.9482614885496183, "accuracy_std_dev": 0.0012429999999999941, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07536056743002545, "avg_duration_per_task_ms": 132814.94147582698, "avg_output_tokens_per_task": 18975.51145038168, "last_run_timestamp": "2026-10-15T21:00:04.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.947917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 96, "avg_cost_per_task_usd": 0.009331666666666667, "avg_duration_per_task_ms": 232878.71875, "avg_output_tokens_per_task": 10278.75, "last_run_timestamp": "2026-10-14T18:00:04.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021931015151515153, "avg_duration_per_task_ms": 68600.9191919192, "avg_output_tokens_per_task": 1835.4242424242425, "last_run_timestamp": "2026-10-03T01:13:05.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11987038888888889, "avg_duration_per_task_ms": 35572.169191919194, "avg_output_tokens_per_task": 2020.9419191919192, "last_run_timestamp": "2026-10-01T08:24:02.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.9393935, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.29793838636363634, "avg_duration_per_task_ms": 106861.7196969697, "avg_output_tokens_per_task": 11692.656565656565, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9391951653543308, "accuracy_std_dev": 0.009884880879403648, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.026313035433070862, "avg_duration_per_task_ms": 56131.107611548556, "avg_output_tokens_per_task": 7729.320209973754, "last_run_timestamp": "2026-10-15T21:01:11.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9371493333333333, "accuracy_std_dev": 0.004199386807089248, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.30925494444444446, "avg_duration_per_task_ms": 152328.37037037036, "avg_output_tokens_per_task": 10179.092592592593, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.9337823333333333, "accuracy_std_dev": 0.008398773614178557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3842955404040404, "avg_duration_per_task_ms": 82918.1936026936, "avg_output_tokens_per_task": 11996.59090909091, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9332206666666667, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.24476150168350166, "avg_duration_per_task_ms": 101605.01851851853, "avg_output_tokens_per_task": 11508.424242424242, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03490100505050505, "avg_duration_per_task_ms": 57907.71885521885, "avg_output_tokens_per_task": 6070.841750841751, "last_run_timestamp": "2026-10-09T23:23:36.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9326596250000001, "accuracy_std_dev": 0.00862529761714779, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.32610833964646463, "avg_duration_per_task_ms": 161836.85416666666, "avg_output_tokens_per_task": 10112.390782828283, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9325395714285714, "accuracy_std_dev": 0.004401689845956867, "run_count": 4, "total_questions": 756, "avg_cost_per_task_usd": 0.06429072222222224, "avg_duration_per_task_ms": 63071.10317460317, "avg_output_tokens_per_task": 11529.871693121693, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9305554166666667, "accuracy_std_dev": 0.007749155464289131, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.026224210437710435, "avg_duration_per_task_ms": 18736.835016835015, "avg_output_tokens_per_task": 2232.077441077441, "last_run_timestamp": "2026-10-17T08:21:08.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.930415, "accuracy_std_dev": 0.004827415699246697, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06216489141414142, "avg_duration_per_task_ms": 64761.851851851854, "avg_output_tokens_per_task": 11730.429292929293, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9295867436026091, "accuracy_std_dev": 0.01014563057765174, "run_count": 11, "total_questions": 1993, "avg_cost_per_task_usd": 0.12675964425489214, "avg_duration_per_task_ms": 135519.84947315603, "avg_output_tokens_per_task": 8382.747114902157, "last_run_timestamp": "2026-10-17T08:31:19.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.92929275, "accuracy_std_dev": 0.003571007097094602, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.024821165404040405, "avg_duration_per_task_ms": 19487.328282828283, "avg_output_tokens_per_task": 2241.7411616161617, "last_run_timestamp": "2026-10-17T08:18:07.776Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9280299999999999, "accuracy_std_dev": 0.0024899934738870384, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025444652777777778, "avg_duration_per_task_ms": 19201.565656565657, "avg_output_tokens_per_task": 2305.0138888888887, "last_run_timestamp": "2026-10-17T08:03:37.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.9276092499999999, "accuracy_std_dev": 0.0031494064817835167, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025043602272727273, "avg_duration_per_task_ms": 20424.821969696968, "avg_output_tokens_per_task": 2271.911616161616, "last_run_timestamp": "2026-10-17T08:18:07.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.004418715424192887, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05958464141414141, "avg_duration_per_task_ms": 58983.43771043771, "avg_output_tokens_per_task": 11740.584175084176, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9264645740259738, "accuracy_std_dev": 0.008111437376199883, "run_count": 31, "total_questions": 5775, "avg_cost_per_task_usd": 0.17016577662337665, "avg_duration_per_task_ms": 157879.1478787879, "avg_output_tokens_per_task": 9136.811948051947, "last_run_timestamp": "2026-10-17T08:23:33.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.9262631428571431, "accuracy_std_dev": 0.007525054247550093, "run_count": 7, "total_questions": 1372, "avg_cost_per_task_usd": 0.11394943294460641, "avg_duration_per_task_ms": 190369.23032069972, "avg_output_tokens_per_task": 9499.49052478134, "last_run_timestamp": "2026-10-17T08:26:17.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9262129285099052, "accuracy_std_dev": 0.013787528526068137, "run_count": 7, "total_questions": 1161, "avg_cost_per_task_usd": 0.10748236175710593, "avg_duration_per_task_ms": 226093.6391042205, "avg_output_tokens_per_task": 8296.273040482343, "last_run_timestamp": "2026-10-17T08:22:58.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.925926, "accuracy_std_dev": 0.005498287975967322, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1441699393939394, "avg_duration_per_task_ms": 61658.986531986535, "avg_output_tokens_per_task": 15894.8367003367, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9259256666666668, "accuracy_std_dev": 0.0036366213562713575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1413088872053872, "avg_duration_per_task_ms": 82468.60606060606, "avg_output_tokens_per_task": 15584.752525252525, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9255440309278352, "accuracy_std_dev": 0.0047577793839843644, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.3407643092783505, "avg_duration_per_task_ms": 132309.46391752578, "avg_output_tokens_per_task": 10197.671821305841, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9248735, "accuracy_std_dev": 0.00862277742377712, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10364464204545455, "avg_duration_per_task_ms": 55123.33648989899, "avg_output_tokens_per_task": 3517.8642676767677, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9242421666666667, "accuracy_std_dev": 0.0048599613653014136, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.14878865656565657, "avg_duration_per_task_ms": 85040.50420875421, "avg_output_tokens_per_task": 15641.649831649831, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto-26.10-preview", "model_slug": "unbiased/pareto-26.10-preview", "model_name": "Pareto 26.10 Preview", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0030723434343434344, "avg_duration_per_task_ms": 37807.181818181816, "avg_output_tokens_per_task": 903.9090909090909, "last_run_timestamp": "2026-10-15T14:24:31.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.924032, "accuracy_std_dev": 0.002938362554212806, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11472872474747475, "avg_duration_per_task_ms": 192645.5883838384, "avg_output_tokens_per_task": 9486.564393939394, "last_run_timestamp": "2026-10-17T08:19:42.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.923681, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025073777777777778, "avg_duration_per_task_ms": 20021.311447811448, "avg_output_tokens_per_task": 2270.7323232323233, "last_run_timestamp": "2026-10-17T08:04:11.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.008625346413333228, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05533751388888889, "avg_duration_per_task_ms": 61705.84722222222, "avg_output_tokens_per_task": 3564.286616161616, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.92312, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1503178552188552, "avg_duration_per_task_ms": 99928.72895622895, "avg_output_tokens_per_task": 9768, "last_run_timestamp": "2026-10-17T08:20:36.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9227054347826086, "accuracy_std_dev": 0.0037342422507786015, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.09927257048748353, "avg_duration_per_task_ms": 52555.73781291173, "avg_output_tokens_per_task": 3495.6416337285905, "last_run_timestamp": "2026-10-01T19:12:57.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9214533832285114, "accuracy_std_dev": 0.007123977352126375, "run_count": 13, "total_questions": 2385, "avg_cost_per_task_usd": 0.14641879832285115, "avg_duration_per_task_ms": 129257.1857442348, "avg_output_tokens_per_task": 9613.907337526205, "last_run_timestamp": "2026-10-17T08:23:01.607Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.9212961249999999, "accuracy_std_dev": 0.006614649375392113, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1696066616161616, "avg_duration_per_task_ms": 303586.860479798, "avg_output_tokens_per_task": 9644.771464646465, "last_run_timestamp": "2026-10-17T08:02:36.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06043574579124578, "avg_duration_per_task_ms": 62971.38047138047, "avg_output_tokens_per_task": 3586.8703703703704, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9200982307692309, "accuracy_std_dev": 0.007282007290405241, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.14751564180264182, "avg_duration_per_task_ms": 191954.80303030304, "avg_output_tokens_per_task": 11844.022144522145, "last_run_timestamp": "2026-10-17T08:26:18.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9196950955414013, "accuracy_std_dev": 0.03543852389220188, "run_count": 14, "total_questions": 2669, "avg_cost_per_task_usd": 0.12421539677781943, "avg_duration_per_task_ms": 189674.27538403898, "avg_output_tokens_per_task": 9508.503559385537, "last_run_timestamp": "2026-10-17T08:31:19.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.9194757752808989, "accuracy_std_dev": 0.005468000000000028, "run_count": 2, "total_questions": 356, "avg_cost_per_task_usd": 0.12247723595505616, "avg_duration_per_task_ms": 247597.07584269662, "avg_output_tokens_per_task": 32087.120786516854, "last_run_timestamp": "2026-10-17T08:23:02.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9193136803377565, "accuracy_std_dev": 0.012748055372296913, "run_count": 13, "total_questions": 2487, "avg_cost_per_task_usd": 0.1257693297145155, "avg_duration_per_task_ms": 119458.23924406916, "avg_output_tokens_per_task": 8396.139123441897, "last_run_timestamp": "2026-10-17T08:19:33.656Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9192753646408839, "accuracy_std_dev": 0.01148102022995612, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.13229984714548804, "avg_duration_per_task_ms": 285821.6473296501, "avg_output_tokens_per_task": 10235.589779005524, "last_run_timestamp": "2026-10-17T08:02:36.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026391550505050505, "avg_duration_per_task_ms": 26392.646464646463, "avg_output_tokens_per_task": 2240.712121212121, "last_run_timestamp": "2026-10-01T08:37:17.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9189512857142857, "accuracy_std_dev": 0.006699457328007703, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.08570814935064935, "avg_duration_per_task_ms": 308581.3556998557, "avg_output_tokens_per_task": 24794.714285714286, "last_run_timestamp": "2026-10-17T08:18:07.429Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9187128800685322, "accuracy_std_dev": 0.014979707918825845, "run_count": 9, "total_questions": 1751, "avg_cost_per_task_usd": 0.24826981896059397, "avg_duration_per_task_ms": 409248.07595659624, "avg_output_tokens_per_task": 36709.836664762995, "last_run_timestamp": "2026-10-17T08:26:18.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9166664999999998, "accuracy_std_dev": 0.003669332602803965, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025965300505050505, "avg_duration_per_task_ms": 19379.844065656565, "avg_output_tokens_per_task": 2237.6660353535353, "last_run_timestamp": "2026-10-17T08:19:39.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9159637816307403, "accuracy_std_dev": 0.011684355360373981, "run_count": 8, "total_questions": 1067, "avg_cost_per_task_usd": 0.11537374320524835, "avg_duration_per_task_ms": 187895.38612933457, "avg_output_tokens_per_task": 7839.027179006561, "last_run_timestamp": "2026-10-17T08:04:21.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9157357830756013, "accuracy_std_dev": 0.007025144934744547, "run_count": 12, "total_questions": 2328, "avg_cost_per_task_usd": 0.14151731056701028, "avg_duration_per_task_ms": 289404.89347079035, "avg_output_tokens_per_task": 9270.16881443299, "last_run_timestamp": "2026-10-17T08:25:47.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.915614375, "accuracy_std_dev": 0.0073109223928568175, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10001511047979798, "avg_duration_per_task_ms": 653810.1868686868, "avg_output_tokens_per_task": 40376.643308080806, "last_run_timestamp": "2026-10-17T08:21:09.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9151233333333333, "accuracy_std_dev": 0.012774219763083605, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03489141035353535, "avg_duration_per_task_ms": 231436.15993265994, "avg_output_tokens_per_task": 28899.603535353537, "last_run_timestamp": "2026-10-17T16:03:42.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9132906754665453, "accuracy_std_dev": 0.029785895483567967, "run_count": 23, "total_questions": 4394, "avg_cost_per_task_usd": 0.14536340896677283, "avg_duration_per_task_ms": 401099.01661356393, "avg_output_tokens_per_task": 8988.59285389167, "last_run_timestamp": "2026-10-18T00:58:02.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9121771666666666, "accuracy_std_dev": 0.008877109240750762, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.15960165235690235, "avg_duration_per_task_ms": 534122.271885522, "avg_output_tokens_per_task": 39986.65572390572, "last_run_timestamp": "2026-10-17T08:20:04.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9117844, "accuracy_std_dev": 0.006079420781620585, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03790721616161616, "avg_duration_per_task_ms": 129456.97070707071, "avg_output_tokens_per_task": 28484.87575757576, "last_run_timestamp": "2026-10-03T09:28:48.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.910344342048864, "accuracy_std_dev": 0.014501798751218183, "run_count": 12, "total_questions": 2333, "avg_cost_per_task_usd": 0.17792582340334334, "avg_duration_per_task_ms": 507774.8448349764, "avg_output_tokens_per_task": 12020.794256322331, "last_run_timestamp": "2026-10-17T08:19:39.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9102327931985295, "accuracy_std_dev": 0.004925353358133757, "run_count": 11, "total_questions": 2176, "avg_cost_per_task_usd": 0.03013682904411765, "avg_duration_per_task_ms": 430481.5505514706, "avg_output_tokens_per_task": 31169.346966911766, "last_run_timestamp": "2026-10-17T16:03:21.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9094274, "accuracy_std_dev": 0.002474202384608023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07117444141414142, "avg_duration_per_task_ms": 135218.17474747475, "avg_output_tokens_per_task": 27695.128282828282, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.9093497692307693, "accuracy_std_dev": 0.015665798833784123, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.1382119296814297, "avg_duration_per_task_ms": 558863.4650349651, "avg_output_tokens_per_task": 40145.44832944833, "last_run_timestamp": "2026-10-17T08:21:36.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0054125, "avg_duration_per_task_ms": 235839.91414141413, "avg_output_tokens_per_task": 10749.717171717171, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9083225551330798, "accuracy_std_dev": 0.025262661104000026, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.037087138994507815, "avg_duration_per_task_ms": 242369.35530207012, "avg_output_tokens_per_task": 30868.7237008872, "last_run_timestamp": "2026-10-17T16:03:14.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9073230999999999, "accuracy_std_dev": 0.016488125499583027, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.03575355858585859, "avg_duration_per_task_ms": 312749.41262626264, "avg_output_tokens_per_task": 29726.523737373736, "last_run_timestamp": "2026-10-17T16:07:08.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.907143, "accuracy_std_dev": null, "run_count": 1, "total_questions": 140, "avg_cost_per_task_usd": 0.012309828571428571, "avg_duration_per_task_ms": 26262.2, "avg_output_tokens_per_task": 10166.814285714287, "last_run_timestamp": "2026-10-17T21:00:07.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9071381724573594, "accuracy_std_dev": 0.03371213456235009, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.015826173089071385, "avg_duration_per_task_ms": 64047.12571067593, "avg_output_tokens_per_task": 3604.423246999368, "last_run_timestamp": "2026-09-30T03:11:21.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.9069864999999999, "accuracy_std_dev": 0.027094409526382632, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.04235188005050505, "avg_duration_per_task_ms": 291380.1031144781, "avg_output_tokens_per_task": 32881.04797979798, "last_run_timestamp": "2026-10-17T16:05:02.601Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9067949999999999, "accuracy_std_dev": 0.021859135265271935, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03907181726354454, "avg_duration_per_task_ms": 267831.84113865934, "avg_output_tokens_per_task": 32752.338842975205, "last_run_timestamp": "2026-10-17T16:03:30.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9067522535599448, "accuracy_std_dev": 0.05130336735714156, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.1400334451079467, "avg_duration_per_task_ms": 309819.79834634817, "avg_output_tokens_per_task": 10155.972439136427, "last_run_timestamp": "2026-10-17T08:18:04.827Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9063747011952191, "accuracy_std_dev": 0.0020234348684023858, "run_count": 3, "total_questions": 502, "avg_cost_per_task_usd": 0.11477266334661354, "avg_duration_per_task_ms": 440554.38844621513, "avg_output_tokens_per_task": 40023.181274900395, "last_run_timestamp": "2026-10-17T08:23:47.110Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.9062234806560137, "accuracy_std_dev": 0.022290388983096536, "run_count": 13, "total_questions": 2378, "avg_cost_per_task_usd": 0.29330102901597976, "avg_duration_per_task_ms": 441358.91000841046, "avg_output_tokens_per_task": 50972.094617325485, "last_run_timestamp": "2026-10-17T16:06:14.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9060298181818183, "accuracy_std_dev": 0.02266404860253019, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.15440814876033057, "avg_duration_per_task_ms": 587765.4531680441, "avg_output_tokens_per_task": 38971.41505968779, "last_run_timestamp": "2026-10-17T08:23:03.040Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.905724, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008862866161616162, "avg_duration_per_task_ms": 144443.7373737374, "avg_output_tokens_per_task": 15989.888888888889, "last_run_timestamp": "2026-10-18T01:07:53.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.905724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06914223232323233, "avg_duration_per_task_ms": 293387.91414141416, "avg_output_tokens_per_task": 16031.914141414141, "last_run_timestamp": "2026-10-17T21:00:07.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9054832857142857, "accuracy_std_dev": 0.008882188721807021, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.021842018037518035, "avg_duration_per_task_ms": 208032.45382395384, "avg_output_tokens_per_task": 29051.58008658009, "last_run_timestamp": "2026-10-17T08:22:58.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.9052889151291513, "accuracy_std_dev": 0.017795645091488572, "run_count": 3, "total_questions": 542, "avg_cost_per_task_usd": 0.12811947416974168, "avg_duration_per_task_ms": 1694586.1826568267, "avg_output_tokens_per_task": 51002.60147601476, "last_run_timestamp": "2026-10-17T08:18:07.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9049022275600507, "accuracy_std_dev": 0.03924926725076816, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.14131352844500636, "avg_duration_per_task_ms": 216359.54614412136, "avg_output_tokens_per_task": 9500.680994521703, "last_run_timestamp": "2026-10-17T08:18:07.439Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9048820000000001, "accuracy_std_dev": 0.0025253333443329903, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03441120328282828, "avg_duration_per_task_ms": 137911.47095959596, "avg_output_tokens_per_task": 28491.76767676768, "last_run_timestamp": "2026-10-17T08:04:21.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.904882, "accuracy_std_dev": 0.00840358433447696, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03428501851851851, "avg_duration_per_task_ms": 234623.59848484848, "avg_output_tokens_per_task": 28392.512626262625, "last_run_timestamp": "2026-10-17T08:18:06.503Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9047746274509804, "accuracy_std_dev": 0.022192155496030565, "run_count": 10, "total_questions": 1836, "avg_cost_per_task_usd": 0.15225744008714598, "avg_duration_per_task_ms": 561798.8300653595, "avg_output_tokens_per_task": 37859.14270152505, "last_run_timestamp": "2026-10-17T08:25:53.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9046852723932015, "accuracy_std_dev": 0.030917632257017453, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.10195215893431328, "avg_duration_per_task_ms": 567988.1175930179, "avg_output_tokens_per_task": 39285.040881947636, "last_run_timestamp": "2026-10-17T08:19:16.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9046213388116308, "accuracy_std_dev": 0.01859967095960451, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.03434321828908554, "avg_duration_per_task_ms": 322804.26085124316, "avg_output_tokens_per_task": 30148.002528445006, "last_run_timestamp": "2026-10-17T16:03:16.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.90446125, "accuracy_std_dev": 0.004353383590668304, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05431423611111111, "avg_duration_per_task_ms": 32466.23232323232, "avg_output_tokens_per_task": 2506.719696969697, "last_run_timestamp": "2026-10-17T08:18:07.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9043450497466605, "accuracy_std_dev": 0.011580572970737592, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.027795939198526028, "avg_duration_per_task_ms": 304566.4965453708, "avg_output_tokens_per_task": 23011.022570244128, "last_run_timestamp": "2026-10-17T16:03:30.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9042808571428572, "accuracy_std_dev": 0.009792912093208319, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03522179004329004, "avg_duration_per_task_ms": 198105.57215007214, "avg_output_tokens_per_task": 29184.305916305915, "last_run_timestamp": "2026-10-17T08:21:10.226Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.9040400000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012735267676767678, "avg_duration_per_task_ms": 67216.78787878787, "avg_output_tokens_per_task": 20958.237373737375, "last_run_timestamp": "2026-10-18T00:00:02.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9038162150401354, "accuracy_std_dev": 0.012954254555025118, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.026641588508660753, "avg_duration_per_task_ms": 188585.9387410224, "avg_output_tokens_per_task": 25722.437262357413, "last_run_timestamp": "2026-10-17T08:18:07.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9030148159392788, "accuracy_std_dev": 0.008815415925383199, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.01550765211891208, "avg_duration_per_task_ms": 411171.4794433903, "avg_output_tokens_per_task": 28853.977229601518, "last_run_timestamp": "2026-10-17T08:30:38.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902918, "accuracy_std_dev": 0.0007933738084913016, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14672305387205387, "avg_duration_per_task_ms": 133620.78956228957, "avg_output_tokens_per_task": 9652.131313131313, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9028881689530687, "accuracy_std_dev": 0.007076458182464086, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.029951561732851982, "avg_duration_per_task_ms": 161666.90108303248, "avg_output_tokens_per_task": 28961.749458483755, "last_run_timestamp": "2026-10-17T08:19:16.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.9027143672230654, "accuracy_std_dev": 0.017487582904735587, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.11330698988366211, "avg_duration_per_task_ms": 583845.9398077896, "avg_output_tokens_per_task": 36138.885179565, "last_run_timestamp": "2026-10-17T08:18:06.586Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.9023565, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11881703282828283, "avg_duration_per_task_ms": 830086.2297979798, "avg_output_tokens_per_task": 47333.19696969697, "last_run_timestamp": "2026-10-17T08:18:06.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9023565, "accuracy_std_dev": 0.0037644204401208687, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03250985984848485, "avg_duration_per_task_ms": 841972.3194444445, "avg_output_tokens_per_task": 37015.48358585859, "last_run_timestamp": "2026-10-17T08:18:07.239Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9020199999999999, "accuracy_std_dev": 0.0038974287934483223, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.053951265656565654, "avg_duration_per_task_ms": 27050.535353535353, "avg_output_tokens_per_task": 2481.211111111111, "last_run_timestamp": "2026-10-17T08:20:30.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.9017853214285716, "accuracy_std_dev": 0.00452549999999996, "run_count": 2, "total_questions": 392, "avg_cost_per_task_usd": 0.03336421428571429, "avg_duration_per_task_ms": 106866.74489795919, "avg_output_tokens_per_task": 27637.576530612245, "last_run_timestamp": "2026-10-17T08:22:12.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.901208909090909, "accuracy_std_dev": 0.005840261800709478, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.20488930509641878, "avg_duration_per_task_ms": 97845.36914600551, "avg_output_tokens_per_task": 7828.393709825528, "last_run_timestamp": "2026-10-17T08:02:02.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.901203, "accuracy_std_dev": null, "run_count": 1, "total_questions": 194, "avg_cost_per_task_usd": 0.01318992268041237, "avg_duration_per_task_ms": 207259.9175257732, "avg_output_tokens_per_task": 21902.201030927834, "last_run_timestamp": "2026-10-17T15:00:06.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.900973165044435, "accuracy_std_dev": 0.019246299929394788, "run_count": 12, "total_questions": 2363, "avg_cost_per_task_usd": 0.11851335124841302, "avg_duration_per_task_ms": 360984.45366060094, "avg_output_tokens_per_task": 8109.957257723233, "last_run_timestamp": "2026-10-17T08:21:12.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9008713529411765, "accuracy_std_dev": 0.005828517351993652, "run_count": 34, "total_questions": 6732, "avg_cost_per_task_usd": 0.21723382456922163, "avg_duration_per_task_ms": 94455.50430778372, "avg_output_tokens_per_task": 7921.044711824124, "last_run_timestamp": "2026-10-17T08:18:12.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9000179188034189, "accuracy_std_dev": 0.00894793549668819, "run_count": 11, "total_questions": 1872, "avg_cost_per_task_usd": 0.02389172542735043, "avg_duration_per_task_ms": 163758.47329059828, "avg_output_tokens_per_task": 19761.337606837606, "last_run_timestamp": "2026-10-17T16:03:32.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8999248888888889, "accuracy_std_dev": 0.006354339085213833, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15479229124579125, "avg_duration_per_task_ms": 130090.1857463524, "avg_output_tokens_per_task": 9547.675645342311, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.8999134797927462, "accuracy_std_dev": 0.016737025742048665, "run_count": 10, "total_questions": 1930, "avg_cost_per_task_usd": 0.014986389637305699, "avg_duration_per_task_ms": 258557.6274611399, "avg_output_tokens_per_task": 18920.399481865286, "last_run_timestamp": "2026-10-17T08:26:18.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8998876666666668, "accuracy_std_dev": 0.004716533340871803, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.05738907373737373, "avg_duration_per_task_ms": 25168.736026936025, "avg_output_tokens_per_task": 2481.9760942760945, "last_run_timestamp": "2026-10-17T08:25:41.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14109027272727273, "avg_duration_per_task_ms": 131303, "avg_output_tokens_per_task": 9278.422558922559, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0412280202020202, "avg_duration_per_task_ms": 141835.40572390571, "avg_output_tokens_per_task": 27211.210437710437, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8995188857142861, "accuracy_std_dev": 0.006231455637198027, "run_count": 35, "total_questions": 6930, "avg_cost_per_task_usd": 0.2226963189033189, "avg_duration_per_task_ms": 94364.85238095238, "avg_output_tokens_per_task": 8111.008658008658, "last_run_timestamp": "2026-10-17T08:26:18.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8995076050632912, "accuracy_std_dev": 0.0054905559391406865, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.013410476793248943, "avg_duration_per_task_ms": 179927.08818565402, "avg_output_tokens_per_task": 26352.98523206751, "last_run_timestamp": "2026-10-17T08:18:05.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8992142666666664, "accuracy_std_dev": 0.004384375584073756, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.05852382592592592, "avg_duration_per_task_ms": 39927.462626262626, "avg_output_tokens_per_task": 2529.2070707070707, "last_run_timestamp": "2026-10-17T08:18:06.772Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8989900000000001, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007148643939393939, "avg_duration_per_task_ms": 224904.53787878787, "avg_output_tokens_per_task": 25295.88888888889, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.005188652155184398, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03249949747474747, "avg_duration_per_task_ms": 137621.66666666666, "avg_output_tokens_per_task": 26156.715909090908, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.0020616538718950853, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03381236616161616, "avg_duration_per_task_ms": 869027.9406565657, "avg_output_tokens_per_task": 38502.118686868685, "last_run_timestamp": "2026-10-17T08:20:32.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8988212999999999, "accuracy_std_dev": 0.004847094842274054, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.22962485303030306, "avg_duration_per_task_ms": 110453.92777777778, "avg_output_tokens_per_task": 8973.717676767677, "last_run_timestamp": "2026-10-17T08:21:05.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.898697092413793, "accuracy_std_dev": 0.020363329103607237, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.12840642850574713, "avg_duration_per_task_ms": 515985.4285057471, "avg_output_tokens_per_task": 38420.705747126434, "last_run_timestamp": "2026-10-17T08:18:00.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.8986529999999999, "accuracy_std_dev": 0.006334877031166399, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.057629573232323233, "avg_duration_per_task_ms": 25925.42575757576, "avg_output_tokens_per_task": 2526.4747474747473, "last_run_timestamp": "2026-10-17T08:19:36.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8985086428571428, "accuracy_std_dev": 0.02010583105045876, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.08497059956709957, "avg_duration_per_task_ms": 213919.4072871573, "avg_output_tokens_per_task": 29834.7113997114, "last_run_timestamp": "2026-10-17T16:03:45.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna", "model_slug": "openai/gpt-6-luna", "model_name": "GPT-6-Luna", "provider_name": "auto-routing", "accuracy": 0.898477, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.003442197969543147, "avg_duration_per_task_ms": 73312.30456852792, "avg_output_tokens_per_task": 6831.9390862944165, "last_run_timestamp": "2026-10-08T13:29:57.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.8977269999999999, "accuracy_std_dev": 0.008938227620731064, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.12509009848484848, "avg_duration_per_task_ms": 997729.1325757576, "avg_output_tokens_per_task": 49776.08964646464, "last_run_timestamp": "2026-10-17T08:18:07.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8974747000000001, "accuracy_std_dev": 0.008159314922835103, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.19849825303030305, "avg_duration_per_task_ms": 95555.76262626263, "avg_output_tokens_per_task": 7731.669191919192, "last_run_timestamp": "2026-10-17T08:22:57.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8973243579175705, "accuracy_std_dev": 0.006747288047801109, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.018802611713665943, "avg_duration_per_task_ms": 445979.23065798986, "avg_output_tokens_per_task": 20953.845986984816, "last_run_timestamp": "2026-10-17T08:23:01.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.8973063333333335, "accuracy_std_dev": 0.007142249894504881, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11625130303030304, "avg_duration_per_task_ms": 577263.1498316498, "avg_output_tokens_per_task": 46324.27272727273, "last_run_timestamp": "2026-10-17T08:04:10.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.897306, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010453515151515153, "avg_duration_per_task_ms": 272743.00505050505, "avg_output_tokens_per_task": 20780.954545454544, "last_run_timestamp": "2026-10-17T15:00:06.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8972450554493306, "accuracy_std_dev": 0.007483975735232924, "run_count": 24, "total_questions": 4707, "avg_cost_per_task_usd": 0.12556499532611007, "avg_duration_per_task_ms": 57794.63182494158, "avg_output_tokens_per_task": 4708.337582324198, "last_run_timestamp": "2026-10-17T08:20:12.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.896873431174089, "accuracy_std_dev": 0.016764191472169333, "run_count": 15, "total_questions": 2964, "avg_cost_per_task_usd": 0.023734335695006745, "avg_duration_per_task_ms": 240258.34311740892, "avg_output_tokens_per_task": 26795.730094466937, "last_run_timestamp": "2026-10-17T08:21:29.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8968568964143425, "accuracy_std_dev": 0.005278733293840445, "run_count": 4, "total_questions": 753, "avg_cost_per_task_usd": 0.15754132403718457, "avg_duration_per_task_ms": 84099.57768924303, "avg_output_tokens_per_task": 9420.674634794157, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.896387909090909, "accuracy_std_dev": 0.011448796653356942, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.20013578328741968, "avg_duration_per_task_ms": 96239.94214876032, "avg_output_tokens_per_task": 7787.9586776859505, "last_run_timestamp": "2026-10-17T08:18:06.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.8962022556962025, "accuracy_std_dev": 0.0022689999999999655, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.020123096202531644, "avg_duration_per_task_ms": 653583.2962025317, "avg_output_tokens_per_task": 42659.516455696205, "last_run_timestamp": "2026-10-10T00:00:04.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8959899189640768, "accuracy_std_dev": 0.02896400676270464, "run_count": 8, "total_questions": 1197, "avg_cost_per_task_usd": 0.024112938178780283, "avg_duration_per_task_ms": 783093.3592314118, "avg_output_tokens_per_task": 22333.22807017544, "last_run_timestamp": "2026-10-17T16:03:42.308Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8959834999999999, "accuracy_std_dev": 0.006828979737120304, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.2064866551226551, "avg_duration_per_task_ms": 100409.85894660895, "avg_output_tokens_per_task": 8045.632034632034, "last_run_timestamp": "2026-10-17T08:13:13.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8957629166666667, "accuracy_std_dev": 0.0073529979425439275, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12094736405723905, "avg_duration_per_task_ms": 58110.515993265995, "avg_output_tokens_per_task": 4618.655303030303, "last_run_timestamp": "2026-10-17T08:18:57.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8952784126849893, "accuracy_std_dev": 0.033903762384218605, "run_count": 12, "total_questions": 2365, "avg_cost_per_task_usd": 0.14386759830866808, "avg_duration_per_task_ms": 806266.6080338267, "avg_output_tokens_per_task": 36028.42875264271, "last_run_timestamp": "2026-10-17T08:18:07.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8951084117647057, "accuracy_std_dev": 0.010289170146741347, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.013490567046173307, "avg_duration_per_task_ms": 308688.6669829222, "avg_output_tokens_per_task": 27618.48513598988, "last_run_timestamp": "2026-10-17T08:20:07.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8951083888888889, "accuracy_std_dev": 0.007379038657943986, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.20431740810886648, "avg_duration_per_task_ms": 70463.12612233446, "avg_output_tokens_per_task": 7440.589225589226, "last_run_timestamp": "2026-10-17T08:23:14.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8950693704171935, "accuracy_std_dev": 0.008974559925376093, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.22676957437842393, "avg_duration_per_task_ms": 108718.53687315634, "avg_output_tokens_per_task": 8847.257901390645, "last_run_timestamp": "2026-10-17T08:04:11.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8950130708661417, "accuracy_std_dev": 0.00891334037763057, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.048151682414698166, "avg_duration_per_task_ms": 43476.24934383202, "avg_output_tokens_per_task": 3503.036745406824, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8949492, "accuracy_std_dev": 0.011447658824405965, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.008737155555555556, "avg_duration_per_task_ms": 172419.46262626263, "avg_output_tokens_per_task": 30724.964646464647, "last_run_timestamp": "2026-10-17T08:26:18.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8942760000000001, "accuracy_std_dev": 0.004441544821343102, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.05451896666666667, "avg_duration_per_task_ms": 31770.72121212121, "avg_output_tokens_per_task": 2522.4050505050504, "last_run_timestamp": "2026-10-17T08:20:03.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8942210023995201, "accuracy_std_dev": 0.013642788754376415, "run_count": 17, "total_questions": 3334, "avg_cost_per_task_usd": 0.003915166466706659, "avg_duration_per_task_ms": 322301.0131973605, "avg_output_tokens_per_task": 21462.84853029394, "last_run_timestamp": "2026-10-17T08:21:32.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8941264444444444, "accuracy_std_dev": 0.009636502052221474, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05178651795735129, "avg_duration_per_task_ms": 45445.343434343435, "avg_output_tokens_per_task": 3671.5897867564536, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.89393925, "accuracy_std_dev": 0.007241007297848803, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1140800441919192, "avg_duration_per_task_ms": 871452.9570707071, "avg_output_tokens_per_task": 45451.416666666664, "last_run_timestamp": "2026-10-17T08:18:04.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.8934709999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 194, "avg_cost_per_task_usd": 0.11686864948453608, "avg_duration_per_task_ms": 605687.7474226804, "avg_output_tokens_per_task": 46496.701030927834, "last_run_timestamp": "2026-10-13T03:00:07.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8928570861678005, "accuracy_std_dev": 0.015340394005370288, "run_count": 5, "total_questions": 882, "avg_cost_per_task_usd": 0.12683532766439912, "avg_duration_per_task_ms": 285532.1031746032, "avg_output_tokens_per_task": 28366.77097505669, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.89267675, "accuracy_std_dev": 0.006979218773437292, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037834488636363635, "avg_duration_per_task_ms": 50710.07575757576, "avg_output_tokens_per_task": 3480.1502525252527, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.892375117408907, "accuracy_std_dev": 0.01166327801187222, "run_count": 15, "total_questions": 2964, "avg_cost_per_task_usd": 0.20794853846153846, "avg_duration_per_task_ms": 100204.1423751687, "avg_output_tokens_per_task": 8089.7665317139, "last_run_timestamp": "2026-10-17T08:04:25.515Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.8922560000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026633252525252525, "avg_duration_per_task_ms": 171876.1313131313, "avg_output_tokens_per_task": 22031.722222222223, "last_run_timestamp": "2026-10-17T19:00:28.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8921505000000001, "accuracy_std_dev": 0.013990169329747226, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.014801192550505052, "avg_duration_per_task_ms": 209365.545770202, "avg_output_tokens_per_task": 24981.652777777777, "last_run_timestamp": "2026-10-17T08:25:25.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8917747142857142, "accuracy_std_dev": 0.02007570557176212, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026639956709956707, "avg_duration_per_task_ms": 151711.15223665224, "avg_output_tokens_per_task": 28656.00072150072, "last_run_timestamp": "2026-10-17T08:26:19.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8914141, "accuracy_std_dev": 0.016939939187907378, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1456811595959596, "avg_duration_per_task_ms": 619546.0101010101, "avg_output_tokens_per_task": 36709.93131313131, "last_run_timestamp": "2026-10-17T08:18:00.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8914076107512954, "accuracy_std_dev": 0.04401694299340168, "run_count": 8, "total_questions": 1544, "avg_cost_per_task_usd": 0.05118517422279792, "avg_duration_per_task_ms": 524717.9449481865, "avg_output_tokens_per_task": 16600.09261658031, "last_run_timestamp": "2026-10-17T16:06:05.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8910815631578948, "accuracy_std_dev": 0.017995824624525182, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.03522420701754386, "avg_duration_per_task_ms": 244594.20964912281, "avg_output_tokens_per_task": 33383.53684210526, "last_run_timestamp": "2026-10-17T08:21:10.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8909229583333332, "accuracy_std_dev": 0.008679762547055289, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.2421371898148148, "avg_duration_per_task_ms": 85114.81292087543, "avg_output_tokens_per_task": 9005.400883838383, "last_run_timestamp": "2026-10-17T08:19:37.698Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.890909, "accuracy_std_dev": 0.009121956215637054, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02464470909090909, "avg_duration_per_task_ms": 256040.4707070707, "avg_output_tokens_per_task": 30465.680808080808, "last_run_timestamp": "2026-10-17T08:18:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.8907719999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 177, "avg_cost_per_task_usd": 0.10733528813559323, "avg_duration_per_task_ms": 969537.395480226, "avg_output_tokens_per_task": 42767.57627118644, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8905723333333333, "accuracy_std_dev": 0.021339306148253502, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13216584343434343, "avg_duration_per_task_ms": 251067.54713804714, "avg_output_tokens_per_task": 43934.003367003366, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8905722857142856, "accuracy_std_dev": 0.010947483868963876, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005104790764790765, "avg_duration_per_task_ms": 226150.0036075036, "avg_output_tokens_per_task": 28015.68181818182, "last_run_timestamp": "2026-10-17T16:06:11.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8905610702341137, "accuracy_std_dev": 0.005541939285123924, "run_count": 5, "total_questions": 897, "avg_cost_per_task_usd": 0.011493041248606466, "avg_duration_per_task_ms": 313351.39576365665, "avg_output_tokens_per_task": 24048.177257525083, "last_run_timestamp": "2026-10-17T08:26:17.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8900633006342494, "accuracy_std_dev": 0.009369877075486213, "run_count": 36, "total_questions": 7095, "avg_cost_per_task_usd": 0.1335988453840733, "avg_duration_per_task_ms": 59373.30049330514, "avg_output_tokens_per_task": 4777.861874559549, "last_run_timestamp": "2026-10-17T08:25:53.452Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.005722799567422281, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.19800758417508418, "avg_duration_per_task_ms": 96141.14141414141, "avg_output_tokens_per_task": 7695.558922558923, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8900111666666666, "accuracy_std_dev": 0.01947983538103498, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02698784511784512, "avg_duration_per_task_ms": 210861.35185185185, "avg_output_tokens_per_task": 27490.997474747473, "last_run_timestamp": "2026-10-17T08:19:16.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8900110000000001, "accuracy_std_dev": 0.0055552329084086844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2164912121212121, "avg_duration_per_task_ms": 104660.01851851853, "avg_output_tokens_per_task": 8434.90404040404, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8895921012658227, "accuracy_std_dev": 0.009804308362766958, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.20471323291139243, "avg_duration_per_task_ms": 99592.4729957806, "avg_output_tokens_per_task": 7961.961603375527, "last_run_timestamp": "2026-10-17T08:06:16.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.8895694008908687, "accuracy_std_dev": 0.0030747750877096573, "run_count": 5, "total_questions": 898, "avg_cost_per_task_usd": 0.03201956347438753, "avg_duration_per_task_ms": 793837.6236080178, "avg_output_tokens_per_task": 36607.681514476615, "last_run_timestamp": "2026-10-17T08:18:02.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8890626292530308, "accuracy_std_dev": 0.008344399302285123, "run_count": 13, "total_questions": 2557, "avg_cost_per_task_usd": 0.015432386781384432, "avg_duration_per_task_ms": 167027.2585060618, "avg_output_tokens_per_task": 26079.107547907704, "last_run_timestamp": "2026-10-17T08:18:06.855Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8886081666666666, "accuracy_std_dev": 0.008877425310239943, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01216526936026936, "avg_duration_per_task_ms": 321417.5909090909, "avg_output_tokens_per_task": 22394.61531986532, "last_run_timestamp": "2026-10-18T01:11:24.163Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8878433783257624, "accuracy_std_dev": 0.020080093052184153, "run_count": 8, "total_questions": 1541, "avg_cost_per_task_usd": 0.155311282284231, "avg_duration_per_task_ms": 605967.6262167423, "avg_output_tokens_per_task": 42247.91953277093, "last_run_timestamp": "2026-10-10T08:22:51.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8877663333333334, "accuracy_std_dev": 0.005555502277522277, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03440870875420875, "avg_duration_per_task_ms": 50195.98148148148, "avg_output_tokens_per_task": 3309.2491582491584, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8875035477214103, "accuracy_std_dev": 0.007722797339695002, "run_count": 12, "total_questions": 2326, "avg_cost_per_task_usd": 0.09761406061908857, "avg_duration_per_task_ms": 475010.2897678418, "avg_output_tokens_per_task": 29146.092863284608, "last_run_timestamp": "2026-10-17T16:03:24.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8871941263616558, "accuracy_std_dev": 0.01904533103531935, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.18794971750181555, "avg_duration_per_task_ms": 1470311.4488017429, "avg_output_tokens_per_task": 55171.43355119826, "last_run_timestamp": "2026-10-17T16:06:45.211Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8867924431103489, "accuracy_std_dev": 0.030089157796681178, "run_count": 9, "total_questions": 1749, "avg_cost_per_task_usd": 0.009832666666666667, "avg_duration_per_task_ms": 319227.98799313896, "avg_output_tokens_per_task": 26332.455117209833, "last_run_timestamp": "2026-10-17T08:31:20.808Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8866412831858407, "accuracy_std_dev": 0.003953963770698987, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.032885610619469034, "avg_duration_per_task_ms": 793000.0455120101, "avg_output_tokens_per_task": 37709.38053097345, "last_run_timestamp": "2026-10-17T08:18:06.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.8863250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.01984508717948718, "avg_duration_per_task_ms": 89498.22564102564, "avg_output_tokens_per_task": 39493.94871794872, "last_run_timestamp": "2026-10-17T08:20:36.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8856619999999997, "accuracy_std_dev": 0.01945709939413717, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.02667407154882155, "avg_duration_per_task_ms": 212253.46506734006, "avg_output_tokens_per_task": 25588.583333333332, "last_run_timestamp": "2026-10-17T08:02:36.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8856209104124406, "accuracy_std_dev": 0.01242517693752844, "run_count": 15, "total_questions": 2958, "avg_cost_per_task_usd": 0.007487724137931034, "avg_duration_per_task_ms": 435881.5027045301, "avg_output_tokens_per_task": 26714.433062880325, "last_run_timestamp": "2026-10-17T08:21:08.586Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8855220000000001, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1415003787878788, "avg_duration_per_task_ms": 184472.2676767677, "avg_output_tokens_per_task": 47043.42424242424, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8855217999999999, "accuracy_std_dev": 0.01517039377735464, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09142711111111111, "avg_duration_per_task_ms": 370562.0878787879, "avg_output_tokens_per_task": 32276.01313131313, "last_run_timestamp": "2026-10-17T08:18:07.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8855217500000001, "accuracy_std_dev": 0.01007759093339443, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.02490623021885522, "avg_duration_per_task_ms": 342831.7175925926, "avg_output_tokens_per_task": 26359.463804713803, "last_run_timestamp": "2026-10-17T08:26:01.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8853686363636363, "accuracy_std_dev": 0.014979663555347446, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.11657416161616162, "avg_duration_per_task_ms": 588624.5243342516, "avg_output_tokens_per_task": 30454.895316804406, "last_run_timestamp": "2026-10-17T16:04:59.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8853190481927712, "accuracy_std_dev": 0.005795847495189989, "run_count": 4, "total_questions": 747, "avg_cost_per_task_usd": 0.0097919718875502, "avg_duration_per_task_ms": 50542.31994645247, "avg_output_tokens_per_task": 7272.5394912985275, "last_run_timestamp": "2026-10-01T19:09:12.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8852812857142857, "accuracy_std_dev": 0.029564788253762207, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1024163391053391, "avg_duration_per_task_ms": 456497.3621933622, "avg_output_tokens_per_task": 36161.62914862915, "last_run_timestamp": "2026-10-17T16:07:02.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8850689656218401, "accuracy_std_dev": 0.014411932672615443, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.026149528816986855, "avg_duration_per_task_ms": 355797.47320525785, "avg_output_tokens_per_task": 26995.41152679474, "last_run_timestamp": "2026-10-10T08:22:45.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8847767229341582, "accuracy_std_dev": 0.013060988043230358, "run_count": 23, "total_questions": 4526, "avg_cost_per_task_usd": 0.19209964803358373, "avg_duration_per_task_ms": 368161.0671674768, "avg_output_tokens_per_task": 31278.666813963766, "last_run_timestamp": "2026-10-17T16:06:12.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8845977213793105, "accuracy_std_dev": 0.003213065076760805, "run_count": 4, "total_questions": 725, "avg_cost_per_task_usd": 0.12423838068965518, "avg_duration_per_task_ms": 424980.20137931034, "avg_output_tokens_per_task": 36853.06344827586, "last_run_timestamp": "2026-10-17T16:06:05.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.8843901497975709, "accuracy_std_dev": 0.009140983682706173, "run_count": 4, "total_questions": 741, "avg_cost_per_task_usd": 0.006282346828609987, "avg_duration_per_task_ms": 497094.5789473684, "avg_output_tokens_per_task": 34511.78542510122, "last_run_timestamp": "2026-10-17T08:18:00.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.884269595505618, "accuracy_std_dev": 0.010583520080008851, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.018860816853932583, "avg_duration_per_task_ms": 74171.62584269662, "avg_output_tokens_per_task": 7862.741573033708, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8838383333333335, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13180945791245793, "avg_duration_per_task_ms": 248970.3754208754, "avg_output_tokens_per_task": 43817.097643097644, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.8838381666666666, "accuracy_std_dev": 0.004007391562960742, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.029480208754208752, "avg_duration_per_task_ms": 87846.02777777778, "avg_output_tokens_per_task": 24419.969696969696, "last_run_timestamp": "2026-10-17T08:26:03.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.8838380000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03186083585858586, "avg_duration_per_task_ms": 197855.25252525252, "avg_output_tokens_per_task": 26300.560606060608, "last_run_timestamp": "2026-10-17T08:21:34.197Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8833967254326782, "accuracy_std_dev": 0.14603517698007254, "run_count": 19, "total_questions": 3409, "avg_cost_per_task_usd": 0.15225260780287475, "avg_duration_per_task_ms": 248086.39131710178, "avg_output_tokens_per_task": 10842.60457612203, "last_run_timestamp": "2026-10-17T08:31:20.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8833942549450549, "accuracy_std_dev": 0.07014329085756449, "run_count": 15, "total_questions": 2730, "avg_cost_per_task_usd": 0.009541483150183149, "avg_duration_per_task_ms": 150939.43516483516, "avg_output_tokens_per_task": 24093.52454212454, "last_run_timestamp": "2026-10-17T08:31:21.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8833333472727273, "accuracy_std_dev": 0.012158692807021237, "run_count": 7, "total_questions": 1100, "avg_cost_per_task_usd": 0.007285390000000001, "avg_duration_per_task_ms": 942880.7490909091, "avg_output_tokens_per_task": 28917.175454545453, "last_run_timestamp": "2026-10-17T08:19:36.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8830752947678703, "accuracy_std_dev": 0.01790514986459119, "run_count": 7, "total_questions": 1357, "avg_cost_per_task_usd": 0.16143720854826824, "avg_duration_per_task_ms": 733318.2763448784, "avg_output_tokens_per_task": 51996.267501842296, "last_run_timestamp": "2026-10-17T16:06:15.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.882979, "accuracy_std_dev": null, "run_count": 1, "total_questions": 188, "avg_cost_per_task_usd": 0.030796239361702126, "avg_duration_per_task_ms": 313329.39361702127, "avg_output_tokens_per_task": 61352.47340425532, "last_run_timestamp": "2026-10-17T08:18:00.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.882767, "accuracy_std_dev": 0.036073332414049926, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.11128402341597796, "avg_duration_per_task_ms": 517321.9977043159, "avg_output_tokens_per_task": 35762.50183654729, "last_run_timestamp": "2026-10-17T08:30:31.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.882724073089701, "accuracy_std_dev": 0.011353269859384124, "run_count": 8, "total_questions": 1505, "avg_cost_per_task_usd": 0.0675646491694352, "avg_duration_per_task_ms": 310422.11893687706, "avg_output_tokens_per_task": 18670.514950166114, "last_run_timestamp": "2026-10-17T16:07:07.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.8821550000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.129083601010101, "avg_duration_per_task_ms": 388751.6590909091, "avg_output_tokens_per_task": 32112.176767676767, "last_run_timestamp": "2026-10-17T16:07:32.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8821546153846153, "accuracy_std_dev": 0.011719582183012278, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.00583898445998446, "avg_duration_per_task_ms": 396669.965034965, "avg_output_tokens_per_task": 29643.133644133643, "last_run_timestamp": "2026-10-17T08:18:07.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8819863000000001, "accuracy_std_dev": 0.01391203304373593, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.18384018737373733, "avg_duration_per_task_ms": 327070.09444444446, "avg_output_tokens_per_task": 46141.83181818182, "last_run_timestamp": "2026-10-17T08:26:18.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8819856633763972, "accuracy_std_dev": 0.010038555521850644, "run_count": 36, "total_questions": 6889, "avg_cost_per_task_usd": 0.13809856582958338, "avg_duration_per_task_ms": 62825.102482218026, "avg_output_tokens_per_task": 5003.652634634926, "last_run_timestamp": "2026-10-17T08:19:33.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.88173375, "accuracy_std_dev": 0.004667830993887862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.041546487373737374, "avg_duration_per_task_ms": 232821.30555555556, "avg_output_tokens_per_task": 36792.617424242424, "last_run_timestamp": "2026-10-17T08:23:33.827Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.881603, "accuracy_std_dev": null, "run_count": 1, "total_questions": 183, "avg_cost_per_task_usd": 0.10363212021857923, "avg_duration_per_task_ms": 876152.9726775957, "avg_output_tokens_per_task": 41457.83606557377, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8815751316455697, "accuracy_std_dev": 0.010764200383163109, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.007927140084388185, "avg_duration_per_task_ms": 240915.00168776372, "avg_output_tokens_per_task": 27983.259071729957, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8813526750788644, "accuracy_std_dev": 0.020871709863832417, "run_count": 5, "total_questions": 951, "avg_cost_per_task_usd": 0.017862513144058886, "avg_duration_per_task_ms": 275442.094637224, "avg_output_tokens_per_task": 29043.33123028391, "last_run_timestamp": "2026-10-17T08:21:08.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8813129999999999, "accuracy_std_dev": 0.008841579779654784, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.031503367003367005, "avg_duration_per_task_ms": 145405.51430976432, "avg_output_tokens_per_task": 26097.064814814814, "last_run_timestamp": "2026-10-17T08:23:28.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.8812964663774403, "accuracy_std_dev": 0.01721654338129465, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.06070962472885033, "avg_duration_per_task_ms": 528815.1655820679, "avg_output_tokens_per_task": 35118.7838033261, "last_run_timestamp": "2026-10-17T16:06:14.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8809836805006587, "accuracy_std_dev": 0.022722285591462856, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.09808666139657445, "avg_duration_per_task_ms": 594163.1073781291, "avg_output_tokens_per_task": 28545.295125164692, "last_run_timestamp": "2026-10-17T08:25:53.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8808310763358778, "accuracy_std_dev": 0.009797982004176503, "run_count": 12, "total_questions": 2358, "avg_cost_per_task_usd": 0.006506368956743002, "avg_duration_per_task_ms": 376147.7417302799, "avg_output_tokens_per_task": 18935.907972858353, "last_run_timestamp": "2026-10-17T08:25:56.789Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8804712857142858, "accuracy_std_dev": 0.01954996565370957, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00817126911976912, "avg_duration_per_task_ms": 383711.4632034632, "avg_output_tokens_per_task": 28807.10028860029, "last_run_timestamp": "2026-10-17T16:06:05.497Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02275759595959596, "avg_duration_per_task_ms": 316937.4393939394, "avg_output_tokens_per_task": 45749.0101010101, "last_run_timestamp": "2026-10-17T08:18:06.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8804433325415677, "accuracy_std_dev": 0.01817374661786, "run_count": 13, "total_questions": 2526, "avg_cost_per_task_usd": 0.12863515320665084, "avg_duration_per_task_ms": 62895.29176563737, "avg_output_tokens_per_task": 4919.452098178939, "last_run_timestamp": "2026-10-17T08:21:06.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8799901428571427, "accuracy_std_dev": 0.018718710398411293, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.22771173593073596, "avg_duration_per_task_ms": 1000338.3607503608, "avg_output_tokens_per_task": 57115.55772005772, "last_run_timestamp": "2026-10-17T16:04:06.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.879189, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.020723571428571428, "avg_duration_per_task_ms": 141095.54497354498, "avg_output_tokens_per_task": 17734.21164021164, "last_run_timestamp": "2026-09-15T00:33:26.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8790683333333332, "accuracy_std_dev": 0.0073650451382791405, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.031043784511784515, "avg_duration_per_task_ms": 126272.08838383839, "avg_output_tokens_per_task": 25679.83080808081, "last_run_timestamp": "2026-10-17T08:23:03.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8790613422382673, "accuracy_std_dev": 0.0067570990696255865, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.01656789963898917, "avg_duration_per_task_ms": 310715.13862815883, "avg_output_tokens_per_task": 29176.26859205776, "last_run_timestamp": "2026-10-17T08:23:40.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8790281428571429, "accuracy_std_dev": 0.029905434553551217, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.19772237878787877, "avg_duration_per_task_ms": 1010773.2835497835, "avg_output_tokens_per_task": 58578.94011544011, "last_run_timestamp": "2026-10-17T16:10:08.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8788940912152681, "accuracy_std_dev": 0.03197805161052743, "run_count": 18, "total_questions": 3563, "avg_cost_per_task_usd": 0.14574238170081394, "avg_duration_per_task_ms": 381005.49284310977, "avg_output_tokens_per_task": 11030.008139208532, "last_run_timestamp": "2026-10-17T08:24:42.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8788589973154363, "accuracy_std_dev": 0.03288382573712152, "run_count": 8, "total_questions": 1490, "avg_cost_per_task_usd": 0.011980427516778525, "avg_duration_per_task_ms": 369898.5476510067, "avg_output_tokens_per_task": 24175.64765100671, "last_run_timestamp": "2026-10-17T08:31:21.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "Deepseek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010643959595959596, "avg_duration_per_task_ms": 127627.76767676767, "avg_output_tokens_per_task": 14277.949494949495, "last_run_timestamp": "2026-10-08T13:29:57.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Mimo V2.6 Pro", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01249009595959596, "avg_duration_per_task_ms": 315799.5808080808, "avg_output_tokens_per_task": 14226.873737373737, "last_run_timestamp": "2026-10-08T01:55:44.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8787876666666666, "accuracy_std_dev": 0.005991714074916775, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007907055555555554, "avg_duration_per_task_ms": 79561.398989899, "avg_output_tokens_per_task": 7834.653198653199, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8786261545293074, "accuracy_std_dev": 0.0068412882173039926, "run_count": 6, "total_questions": 1126, "avg_cost_per_task_usd": 0.11316016607460035, "avg_duration_per_task_ms": 476858.84191829484, "avg_output_tokens_per_task": 32318.380994671403, "last_run_timestamp": "2026-10-17T16:07:16.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8785058930232558, "accuracy_std_dev": 0.05081114611234429, "run_count": 12, "total_questions": 2365, "avg_cost_per_task_usd": 0.0752388731501057, "avg_duration_per_task_ms": 545552.287103594, "avg_output_tokens_per_task": 39460.21268498943, "last_run_timestamp": "2026-10-17T16:06:05.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8783141613793103, "accuracy_std_dev": 0.007309229033068505, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.11746212551724139, "avg_duration_per_task_ms": 746972.4225287356, "avg_output_tokens_per_task": 30792.3508045977, "last_run_timestamp": "2026-10-17T16:07:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8779042927756654, "accuracy_std_dev": 0.017419794249287197, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.03390838529784537, "avg_duration_per_task_ms": 751391.9290240811, "avg_output_tokens_per_task": 39097.844106463876, "last_run_timestamp": "2026-10-17T08:18:06.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8776864563843236, "accuracy_std_dev": 0.005946450694269239, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.12355401517067002, "avg_duration_per_task_ms": 906793.7718078382, "avg_output_tokens_per_task": 43362.45132743363, "last_run_timestamp": "2026-10-17T16:03:16.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.8773939383465671, "accuracy_std_dev": 0.06611258607459759, "run_count": 11, "total_questions": 2141, "avg_cost_per_task_usd": 0.011637005137786084, "avg_duration_per_task_ms": 466296.29145259224, "avg_output_tokens_per_task": 22336.07753386268, "last_run_timestamp": "2026-10-17T08:31:20.282Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8771042727272729, "accuracy_std_dev": 0.017480755975388337, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.13006525436179983, "avg_duration_per_task_ms": 63287.73140495868, "avg_output_tokens_per_task": 4980.730027548209, "last_run_timestamp": "2026-10-17T08:04:10.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.877104, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007626962121212121, "avg_duration_per_task_ms": 696482.2146464647, "avg_output_tokens_per_task": 43429.71717171717, "last_run_timestamp": "2026-10-17T16:06:09.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8765007667238421, "accuracy_std_dev": 0.009087011145341243, "run_count": 3, "total_questions": 583, "avg_cost_per_task_usd": 0.09344886277873071, "avg_duration_per_task_ms": 568169.3550600343, "avg_output_tokens_per_task": 27396.61578044597, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8759949037313433, "accuracy_std_dev": 0.017795644369637542, "run_count": 7, "total_questions": 1340, "avg_cost_per_task_usd": 0.18478927313432836, "avg_duration_per_task_ms": 777556.180597015, "avg_output_tokens_per_task": 53320.65, "last_run_timestamp": "2026-10-17T16:05:59.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.8755271341772152, "accuracy_std_dev": 0.059505259340153195, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.08042012827004219, "avg_duration_per_task_ms": 714967.8244725738, "avg_output_tokens_per_task": 30676.93164556962, "last_run_timestamp": "2026-10-17T16:06:09.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.87542075, "accuracy_std_dev": 0.004610637205148564, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0796655340909091, "avg_duration_per_task_ms": 373448.2607323232, "avg_output_tokens_per_task": 22652.47537878788, "last_run_timestamp": "2026-10-17T16:07:05.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04156198232323233, "avg_duration_per_task_ms": 309570.8888888889, "avg_output_tokens_per_task": 39159.9494949495, "last_run_timestamp": "2026-10-17T08:26:18.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8751617692307694, "accuracy_std_dev": 0.014328220895270583, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.11090025951825952, "avg_duration_per_task_ms": 290979.16355866357, "avg_output_tokens_per_task": 31543.16822066822, "last_run_timestamp": "2026-10-17T16:03:21.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8750232023608768, "accuracy_std_dev": 0.011627604386879436, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.009395388420460931, "avg_duration_per_task_ms": 434284.4266441821, "avg_output_tokens_per_task": 22384.09162450815, "last_run_timestamp": "2026-10-17T08:31:21.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8749014196217493, "accuracy_std_dev": 0.02259480542425625, "run_count": 5, "total_questions": 846, "avg_cost_per_task_usd": 0.12702199408983453, "avg_duration_per_task_ms": 548158.9716312056, "avg_output_tokens_per_task": 48997.07092198582, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8744278861209964, "accuracy_std_dev": 0.02518065250226848, "run_count": 10, "total_questions": 1967, "avg_cost_per_task_usd": 0.08236600203355363, "avg_duration_per_task_ms": 405834.9110320285, "avg_output_tokens_per_task": 31974.98373157092, "last_run_timestamp": "2026-10-17T16:03:24.607Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8742985, "accuracy_std_dev": 0.016485202058715166, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03491330218855219, "avg_duration_per_task_ms": 143971.7685185185, "avg_output_tokens_per_task": 26443.413299663298, "last_run_timestamp": "2026-10-17T08:21:10.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8741963636363635, "accuracy_std_dev": 0.03122712595651627, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.16790697474747474, "avg_duration_per_task_ms": 351374.1510560147, "avg_output_tokens_per_task": 38330.46235078053, "last_run_timestamp": "2026-10-17T16:07:34.585Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.8741024915384615, "accuracy_std_dev": 0.03167681555707161, "run_count": 7, "total_questions": 1300, "avg_cost_per_task_usd": 0.20500766230769232, "avg_duration_per_task_ms": 1300362.693076923, "avg_output_tokens_per_task": 51104.55923076923, "last_run_timestamp": "2026-10-17T16:05:57.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8740044607508533, "accuracy_std_dev": 0.007605951280996289, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.10401818771331058, "avg_duration_per_task_ms": 459122.319112628, "avg_output_tokens_per_task": 30022.856655290103, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8739477499999999, "accuracy_std_dev": 0.015161742633599224, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.020946954545454546, "avg_duration_per_task_ms": 723886.8560606061, "avg_output_tokens_per_task": 35575.18244949495, "last_run_timestamp": "2026-10-17T16:05:38.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01707317171717172, "avg_duration_per_task_ms": 108581.24747474748, "avg_output_tokens_per_task": 42482.4898989899, "last_run_timestamp": "2026-10-17T08:04:11.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.8733865833333335, "accuracy_std_dev": 0.012291988600021385, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0063178345959595955, "avg_duration_per_task_ms": 267411.5660774411, "avg_output_tokens_per_task": 18551.484427609426, "last_run_timestamp": "2026-10-17T08:18:06.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8733164166666666, "accuracy_std_dev": 0.045719398832184864, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09441799915824915, "avg_duration_per_task_ms": 629233.5349326599, "avg_output_tokens_per_task": 41849.718434343435, "last_run_timestamp": "2026-10-17T16:07:29.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8732769164556962, "accuracy_std_dev": 0.0036989960650004694, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.04468822784810126, "avg_duration_per_task_ms": 182929.39156118143, "avg_output_tokens_per_task": 38802.67004219409, "last_run_timestamp": "2026-10-17T08:22:53.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8732727052578362, "accuracy_std_dev": 0.015296519341340365, "run_count": 10, "total_questions": 1978, "avg_cost_per_task_usd": 0.09049616683518706, "avg_duration_per_task_ms": 212094.93225480284, "avg_output_tokens_per_task": 22916.149140546007, "last_run_timestamp": "2026-10-17T08:21:14.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8731974290076335, "accuracy_std_dev": 0.011081761001303017, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.04679652417302798, "avg_duration_per_task_ms": 227836.92264631044, "avg_output_tokens_per_task": 34673.968956743, "last_run_timestamp": "2026-10-17T08:26:00.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.873136, "accuracy_std_dev": 0.06760829277325589, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.12835601948051947, "avg_duration_per_task_ms": 1515824.0468975469, "avg_output_tokens_per_task": 55051.631313131315, "last_run_timestamp": "2026-10-17T16:05:37.599Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8728152090966519, "accuracy_std_dev": 0.007321385674003263, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.04953204042956412, "avg_duration_per_task_ms": 276172.8900821225, "avg_output_tokens_per_task": 21040.667719519897, "last_run_timestamp": "2026-10-17T16:03:30.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8727272475524476, "accuracy_std_dev": 0.029233686851601524, "run_count": 8, "total_questions": 1430, "avg_cost_per_task_usd": 0.09700304615384615, "avg_duration_per_task_ms": 194874.37272727274, "avg_output_tokens_per_task": 26397.99300699301, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8724903310344827, "accuracy_std_dev": 0.10049535477303755, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.02776077609195403, "avg_duration_per_task_ms": 206156.86804597703, "avg_output_tokens_per_task": 23904.91816091954, "last_run_timestamp": "2026-10-17T16:03:16.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8724745, "accuracy_std_dev": 0.008858340547190563, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008325782828282828, "avg_duration_per_task_ms": 77307.88005050505, "avg_output_tokens_per_task": 7842.704545454545, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.872399359223301, "accuracy_std_dev": 0.037525025908999386, "run_count": 11, "total_questions": 2163, "avg_cost_per_task_usd": 0.10075985020804439, "avg_duration_per_task_ms": 612085.7415626445, "avg_output_tokens_per_task": 25038.095238095237, "last_run_timestamp": "2026-10-17T16:03:14.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8723343333333334, "accuracy_std_dev": 0.005338803632109208, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06334109932659931, "avg_duration_per_task_ms": 262990.18855218857, "avg_output_tokens_per_task": 21026.934343434343, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8721831538461537, "accuracy_std_dev": 0.016072645308321247, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.008282733100233098, "avg_duration_per_task_ms": 224627.168997669, "avg_output_tokens_per_task": 29323.653457653458, "last_run_timestamp": "2026-10-17T08:23:45.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8721539397539244, "accuracy_std_dev": 0.008970575607073516, "run_count": 12, "total_questions": 2357, "avg_cost_per_task_usd": 0.008344311412812899, "avg_duration_per_task_ms": 339433.18073822657, "avg_output_tokens_per_task": 29494.21552821383, "last_run_timestamp": "2026-10-17T08:23:51.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8720540000000001, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004076121212121212, "avg_duration_per_task_ms": 39248.412457912455, "avg_output_tokens_per_task": 7919.883838383838, "last_run_timestamp": "2026-10-17T08:19:02.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8720537777777777, "accuracy_std_dev": 0.005143031980322249, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.004249169472502806, "avg_duration_per_task_ms": 39560.52076318743, "avg_output_tokens_per_task": 7734.946689113356, "last_run_timestamp": "2026-10-17T08:18:06.937Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8718132857142857, "accuracy_std_dev": 0.025315326948111786, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14662903607503608, "avg_duration_per_task_ms": 951998.4913419914, "avg_output_tokens_per_task": 61296.518037518035, "last_run_timestamp": "2026-10-17T16:03:24.607Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8714859264426126, "accuracy_std_dev": 0.00752103815972237, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.06945589727330374, "avg_duration_per_task_ms": 276858.71528218134, "avg_output_tokens_per_task": 11308.629676601142, "last_run_timestamp": "2026-10-10T08:22:14.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8713910913385827, "accuracy_std_dev": 0.02908788258380426, "run_count": 14, "total_questions": 2540, "avg_cost_per_task_usd": 0.011632457874015747, "avg_duration_per_task_ms": 341476.78307086613, "avg_output_tokens_per_task": 20755.323622047243, "last_run_timestamp": "2026-10-17T08:18:00.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8709067008816122, "accuracy_std_dev": 0.03126206183140546, "run_count": 9, "total_questions": 1588, "avg_cost_per_task_usd": 0.024239201511335014, "avg_duration_per_task_ms": 368035.88853904285, "avg_output_tokens_per_task": 24202.87468513854, "last_run_timestamp": "2026-10-17T16:03:19.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.870791, "accuracy_std_dev": 0.011807038716375923, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.08650412815656565, "avg_duration_per_task_ms": 201299.91603535353, "avg_output_tokens_per_task": 14085.986742424242, "last_run_timestamp": "2026-10-17T08:18:06.671Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8706306660808436, "accuracy_std_dev": 0.013549251556023127, "run_count": 9, "total_questions": 1707, "avg_cost_per_task_usd": 0.008233070298769771, "avg_duration_per_task_ms": 238186.8295254833, "avg_output_tokens_per_task": 29547.44698301113, "last_run_timestamp": "2026-10-17T16:03:22.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.8703702499999999, "accuracy_std_dev": 0.009143745481885412, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07626149684343433, "avg_duration_per_task_ms": 105007.26452020202, "avg_output_tokens_per_task": 12387.4273989899, "last_run_timestamp": "2026-10-17T08:03:37.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.8702065335018964, "accuracy_std_dev": 0.007070779465518657, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.31242882300884955, "avg_duration_per_task_ms": 1310479.2250316055, "avg_output_tokens_per_task": 52210.361567635904, "last_run_timestamp": "2026-10-17T16:03:45.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.870157356321839, "accuracy_std_dev": 0.01848232110693894, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.07430513729246488, "avg_duration_per_task_ms": 401768.99042145594, "avg_output_tokens_per_task": 23634.381864623243, "last_run_timestamp": "2026-10-17T08:20:36.736Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8699560401315789, "accuracy_std_dev": 0.0072051253688084314, "run_count": 9, "total_questions": 1520, "avg_cost_per_task_usd": 0.05408648092105263, "avg_duration_per_task_ms": 430159.67631578946, "avg_output_tokens_per_task": 17930.499342105264, "last_run_timestamp": "2026-10-17T16:03:54.454Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8698892857142857, "accuracy_std_dev": 0.00522506975248824, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008162125541125541, "avg_duration_per_task_ms": 376919.626984127, "avg_output_tokens_per_task": 28806.24891774892, "last_run_timestamp": "2026-10-17T16:06:10.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.8692479999999999, "accuracy_std_dev": 0.006780740569190578, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003940161616161617, "avg_duration_per_task_ms": 37433.5505050505, "avg_output_tokens_per_task": 7640.225589225589, "last_run_timestamp": "2026-10-17T08:20:54.898Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8692479999999999, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22171807575757574, "avg_duration_per_task_ms": 260359.50673400675, "avg_output_tokens_per_task": 22076.11111111111, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024268303030303032, "avg_duration_per_task_ms": 252794.202020202, "avg_output_tokens_per_task": 20048.777777777777, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8686612136752137, "accuracy_std_dev": 0.02320764839922872, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.025878321367521367, "avg_duration_per_task_ms": 786906.9829059829, "avg_output_tokens_per_task": 23783.8, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.86847625, "accuracy_std_dev": 0.018488540753328793, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19820480492424242, "avg_duration_per_task_ms": 528317.1729797979, "avg_output_tokens_per_task": 57987.417297979795, "last_run_timestamp": "2026-10-17T16:03:44.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8684461428571428, "accuracy_std_dev": 0.017689481203961483, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.037552243867243866, "avg_duration_per_task_ms": 502237.20707070705, "avg_output_tokens_per_task": 34877.69985569986, "last_run_timestamp": "2026-10-17T16:07:06.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.868376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.027671912820512817, "avg_duration_per_task_ms": 285303.4820512821, "avg_output_tokens_per_task": 21385.076923076922, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8682058571428571, "accuracy_std_dev": 0.007673379706651383, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07688844300144301, "avg_duration_per_task_ms": 83582.13492063493, "avg_output_tokens_per_task": 12574.14357864358, "last_run_timestamp": "2026-10-10T08:21:46.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.8681256666666667, "accuracy_std_dev": 0.00619849967510062, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04747902525252525, "avg_duration_per_task_ms": 225031.7861952862, "avg_output_tokens_per_task": 39421.97306397306, "last_run_timestamp": "2026-10-17T08:26:17.952Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8679384444444443, "accuracy_std_dev": 0.0043547631932328865, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0705021952861953, "avg_duration_per_task_ms": 234442.81986531988, "avg_output_tokens_per_task": 11483.022446689114, "last_run_timestamp": "2026-10-17T08:04:10.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8679118212141426, "accuracy_std_dev": 0.029083773117496287, "run_count": 8, "total_questions": 1499, "avg_cost_per_task_usd": 0.011126581054036026, "avg_duration_per_task_ms": 399635.85390260175, "avg_output_tokens_per_task": 24844.47164776518, "last_run_timestamp": "2026-10-17T08:26:17.731Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8677515555555556, "accuracy_std_dev": 0.008841170600109865, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0042902659932659935, "avg_duration_per_task_ms": 35789.490460157125, "avg_output_tokens_per_task": 7820.8361391694725, "last_run_timestamp": "2026-10-17T08:26:18.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8677245714285714, "accuracy_std_dev": 0.02218605746767963, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11909902236652237, "avg_duration_per_task_ms": 600755.6313131313, "avg_output_tokens_per_task": 35283.31168831169, "last_run_timestamp": "2026-10-17T16:04:59.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.867634625, "accuracy_std_dev": 0.007431114989311851, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025694838383838382, "avg_duration_per_task_ms": 248942.36553030304, "avg_output_tokens_per_task": 21167.008207070707, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.8672838333333334, "accuracy_std_dev": 0.005618689361309189, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004177218855218855, "avg_duration_per_task_ms": 35769.420875420874, "avg_output_tokens_per_task": 7720.202861952862, "last_run_timestamp": "2026-10-17T08:26:01.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8663297, "accuracy_std_dev": 0.04012380805220261, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1796413303030303, "avg_duration_per_task_ms": 45593.2398989899, "avg_output_tokens_per_task": 3390.838888888889, "last_run_timestamp": "2026-10-17T08:18:03.745Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.8661615, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0040270404040404035, "avg_duration_per_task_ms": 38264.56565656565, "avg_output_tokens_per_task": 7821.176767676768, "last_run_timestamp": "2026-10-17T08:17:57.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.865880181448332, "accuracy_std_dev": 0.018202091806230666, "run_count": 7, "total_questions": 1229, "avg_cost_per_task_usd": 0.005052348250610252, "avg_duration_per_task_ms": 670227.6916192026, "avg_output_tokens_per_task": 27781.558991049635, "last_run_timestamp": "2026-10-17T16:03:21.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8655302499999998, "accuracy_std_dev": 0.017156845031575593, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.038863957702020197, "avg_duration_per_task_ms": 732278.3465909091, "avg_output_tokens_per_task": 34126.10921717172, "last_run_timestamp": "2026-10-17T16:03:30.488Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8654881000000001, "accuracy_std_dev": 0.027275302500430684, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007844937878787879, "avg_duration_per_task_ms": 473405.9202020202, "avg_output_tokens_per_task": 25350.743434343436, "last_run_timestamp": "2026-10-17T08:18:05.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8653196666666667, "accuracy_std_dev": 0.008584133982075423, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07009036139169472, "avg_duration_per_task_ms": 231708.10998877665, "avg_output_tokens_per_task": 11412.423681257014, "last_run_timestamp": "2026-10-17T08:19:03.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.865101093922652, "accuracy_std_dev": 0.005917000000000006, "run_count": 2, "total_questions": 362, "avg_cost_per_task_usd": 0.005541756906077348, "avg_duration_per_task_ms": 151652.73480662983, "avg_output_tokens_per_task": 19668.124309392264, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.8648990000000001, "accuracy_std_dev": 0.017388735477314038, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07359775946969697, "avg_duration_per_task_ms": 279722.81755050505, "avg_output_tokens_per_task": 20944.539772727272, "last_run_timestamp": "2026-10-17T16:03:30.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.8646462, "accuracy_std_dev": 0.005985377628855152, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.023131965656565653, "avg_duration_per_task_ms": 316631.1070707071, "avg_output_tokens_per_task": 37590.83535353535, "last_run_timestamp": "2026-10-17T08:22:58.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8640821969539633, "accuracy_std_dev": 0.024524497231589105, "run_count": 15, "total_questions": 2889, "avg_cost_per_task_usd": 0.03844685289027345, "avg_duration_per_task_ms": 78089.81827622015, "avg_output_tokens_per_task": 24843.67185877466, "last_run_timestamp": "2026-10-17T08:19:08.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8639263412162163, "accuracy_std_dev": 0.027057887036664362, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.10281788851351349, "avg_duration_per_task_ms": 359495.01407657657, "avg_output_tokens_per_task": 23127.624436936938, "last_run_timestamp": "2026-10-17T08:31:21.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8637523318442885, "accuracy_std_dev": 0.013965236766252836, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.07525077600510531, "avg_duration_per_task_ms": 273604.328653478, "avg_output_tokens_per_task": 20778.284620293554, "last_run_timestamp": "2026-10-17T16:07:17.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.863636, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22125699158249157, "avg_duration_per_task_ms": 259075.24915824915, "avg_output_tokens_per_task": 22030.511784511786, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8635951463414634, "accuracy_std_dev": 0.01668149999999996, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.047886756097560976, "avg_duration_per_task_ms": 401842.43631436315, "avg_output_tokens_per_task": 25387.241192411922, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8634711176470588, "accuracy_std_dev": 0.012638435637372207, "run_count": 5, "total_questions": 918, "avg_cost_per_task_usd": 0.02367367320261438, "avg_duration_per_task_ms": 131839.07298474945, "avg_output_tokens_per_task": 36232.9880174292, "last_run_timestamp": "2026-10-17T08:21:32.495Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8634435114503817, "accuracy_std_dev": 0.01366835982067709, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.13531842366412217, "avg_duration_per_task_ms": 289599.17048346053, "avg_output_tokens_per_task": 28034.88358778626, "last_run_timestamp": "2026-10-17T08:23:25.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.8630240909090909, "accuracy_std_dev": 0.013961714569457724, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7496591460055096, "avg_duration_per_task_ms": 494920.16850321396, "avg_output_tokens_per_task": 29823.61570247934, "last_run_timestamp": "2026-10-17T08:18:05.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8624119090909091, "accuracy_std_dev": 0.008735668278069101, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8035566230486685, "avg_duration_per_task_ms": 460560.63590449956, "avg_output_tokens_per_task": 31970.262626262625, "last_run_timestamp": "2026-10-17T08:02:36.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8621932857142858, "accuracy_std_dev": 0.007166829677943584, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004471386002886003, "avg_duration_per_task_ms": 249807.05122655124, "avg_output_tokens_per_task": 29262.619047619046, "last_run_timestamp": "2026-10-17T16:06:17.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.861993011958763, "accuracy_std_dev": 0.030079142720378513, "run_count": 13, "total_questions": 2425, "avg_cost_per_task_usd": 0.02418231587628866, "avg_duration_per_task_ms": 455044.95463917527, "avg_output_tokens_per_task": 23028.103917525772, "last_run_timestamp": "2026-10-17T08:18:04.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8617332608695653, "accuracy_std_dev": 0.02687779012852092, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.1588332134387352, "avg_duration_per_task_ms": 153168.5540184453, "avg_output_tokens_per_task": 15408.788537549408, "last_run_timestamp": "2026-10-17T08:22:59.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8616467272727273, "accuracy_std_dev": 0.00836578286388189, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8426584527089072, "avg_duration_per_task_ms": 413263.7479338843, "avg_output_tokens_per_task": 33515.24885215794, "last_run_timestamp": "2026-10-17T08:03:37.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8616161, "accuracy_std_dev": 0.0180033848064746, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09928383787878788, "avg_duration_per_task_ms": 481966.9338383838, "avg_output_tokens_per_task": 26359.05101010101, "last_run_timestamp": "2026-10-17T16:03:19.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8615345381097562, "accuracy_std_dev": 0.03207393262697919, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.13308645274390243, "avg_duration_per_task_ms": 545136.4395325204, "avg_output_tokens_per_task": 42522.091971544716, "last_run_timestamp": "2026-10-17T16:06:11.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8615317499999998, "accuracy_std_dev": 0.035989945114631224, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1836229675925926, "avg_duration_per_task_ms": 45563.180555555555, "avg_output_tokens_per_task": 3466.4579124579122, "last_run_timestamp": "2026-10-17T08:18:07.803Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8614476, "accuracy_std_dev": 0.0162708383754495, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.08960124141414141, "avg_duration_per_task_ms": 187539.3025252525, "avg_output_tokens_per_task": 24757.361616161616, "last_run_timestamp": "2026-10-17T16:04:59.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8613916666666668, "accuracy_std_dev": 0.05599875111504226, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10855281649831651, "avg_duration_per_task_ms": 469370.49326599325, "avg_output_tokens_per_task": 27476.757575757576, "last_run_timestamp": "2026-10-17T16:03:19.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8613916666666666, "accuracy_std_dev": 0.0411070840839981, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.17600762920875418, "avg_duration_per_task_ms": 43642.34217171717, "avg_output_tokens_per_task": 3313.9747474747473, "last_run_timestamp": "2026-10-17T08:04:21.178Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8610461538461541, "accuracy_std_dev": 0.012042697039132513, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.8432189724164725, "avg_duration_per_task_ms": 455003.31235431234, "avg_output_tokens_per_task": 33550.2871017871, "last_run_timestamp": "2026-10-17T08:20:29.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8610345454545455, "accuracy_std_dev": 0.007875541064975502, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.8829572610192835, "avg_duration_per_task_ms": 415451.3753443526, "avg_output_tokens_per_task": 33445.12649219467, "last_run_timestamp": "2026-10-17T08:02:36.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8609005, "accuracy_std_dev": 0.01279172761983305, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.009262198232323233, "avg_duration_per_task_ms": 325442.8162878788, "avg_output_tokens_per_task": 32760.926767676767, "last_run_timestamp": "2026-10-17T16:03:24.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8608923543307087, "accuracy_std_dev": 0.017439499999999997, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.003837814960629921, "avg_duration_per_task_ms": 298144.1771653543, "avg_output_tokens_per_task": 11523.807086614173, "last_run_timestamp": "2026-10-03T17:04:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.8606902500000001, "accuracy_std_dev": 0.008858447447916569, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07241029419191919, "avg_duration_per_task_ms": 429769.84722222225, "avg_output_tokens_per_task": 56551.03535353536, "last_run_timestamp": "2026-10-17T16:07:34.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.859963090909091, "accuracy_std_dev": 0.010045046128266804, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8526076652892564, "avg_duration_per_task_ms": 482177.8953168044, "avg_output_tokens_per_task": 33913.51193755739, "last_run_timestamp": "2026-10-17T08:18:00.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8598952222222221, "accuracy_std_dev": 0.0069538031605058765, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.06597872390572392, "avg_duration_per_task_ms": 219441.50561167227, "avg_output_tokens_per_task": 10728.320426487093, "last_run_timestamp": "2026-10-17T08:18:06.079Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8597882857142857, "accuracy_std_dev": 0.02839544075936078, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.12813852813852813, "avg_duration_per_task_ms": 499282.8939393939, "avg_output_tokens_per_task": 33659.728715728714, "last_run_timestamp": "2026-10-17T16:03:22.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8596491924664603, "accuracy_std_dev": 0.037215045235596586, "run_count": 20, "total_questions": 3876, "avg_cost_per_task_usd": 0.004361001805985552, "avg_duration_per_task_ms": 88288.53689370485, "avg_output_tokens_per_task": 16394.82894736842, "last_run_timestamp": "2026-10-17T08:02:02.308Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8592592, "accuracy_std_dev": 0.0373207358952098, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.12732033636363638, "avg_duration_per_task_ms": 630447.8414141415, "avg_output_tokens_per_task": 42741.22323232323, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8592169999999999, "accuracy_std_dev": 0.023807429874936097, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0034952645202020204, "avg_duration_per_task_ms": 466987.4917929293, "avg_output_tokens_per_task": 33328.60227272727, "last_run_timestamp": "2026-10-17T16:06:34.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8591320634715027, "accuracy_std_dev": 0.0170592919090301, "run_count": 8, "total_questions": 1544, "avg_cost_per_task_usd": 0.08052118911917099, "avg_duration_per_task_ms": 345606.61269430054, "avg_output_tokens_per_task": 27437.18847150259, "last_run_timestamp": "2026-10-17T08:18:00.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8589818529411762, "accuracy_std_dev": 0.027213277180268808, "run_count": 34, "total_questions": 6732, "avg_cost_per_task_usd": 0.1721233055555556, "avg_duration_per_task_ms": 158539.5931372549, "avg_output_tokens_per_task": 16076.290106951872, "last_run_timestamp": "2026-10-17T08:22:59.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8586479490968801, "accuracy_std_dev": 0.00913590390487393, "run_count": 22, "total_questions": 3654, "avg_cost_per_task_usd": 0.17185495894909686, "avg_duration_per_task_ms": 250875.80268199233, "avg_output_tokens_per_task": 31401.503831417624, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8585856363636363, "accuracy_std_dev": 0.019208303437802044, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7565819077134986, "avg_duration_per_task_ms": 502366.63636363635, "avg_output_tokens_per_task": 30138.17401285583, "last_run_timestamp": "2026-10-17T08:04:11.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8583754485087226, "accuracy_std_dev": 0.028677200239136932, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.00778828474957794, "avg_duration_per_task_ms": 416499.75858187955, "avg_output_tokens_per_task": 25916.474957794035, "last_run_timestamp": "2026-10-17T08:25:56.871Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8580637958148383, "accuracy_std_dev": 0.014779169880591208, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.08286067279644893, "avg_duration_per_task_ms": 195379.60684844642, "avg_output_tokens_per_task": 25387.850348763473, "last_run_timestamp": "2026-10-17T08:26:18.335Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8576645647133758, "accuracy_std_dev": 0.020977893625656868, "run_count": 40, "total_questions": 7850, "avg_cost_per_task_usd": 0.27657329388535035, "avg_duration_per_task_ms": 335319.63796178345, "avg_output_tokens_per_task": 49792.269044585984, "last_run_timestamp": "2026-10-17T16:06:05.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8576035833333334, "accuracy_std_dev": 0.030166058535652105, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1556503026094276, "avg_duration_per_task_ms": 151996.30134680134, "avg_output_tokens_per_task": 15348.492003367004, "last_run_timestamp": "2026-10-17T08:19:33.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8574634166666667, "accuracy_std_dev": 0.039912043761790195, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.18122077314814813, "avg_duration_per_task_ms": 46037.724326599324, "avg_output_tokens_per_task": 3416.7558922558924, "last_run_timestamp": "2026-10-17T08:19:36.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.857416679028133, "accuracy_std_dev": 0.024059550515011203, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.1341864226342711, "avg_duration_per_task_ms": 371153.3861892583, "avg_output_tokens_per_task": 30475.74872122762, "last_run_timestamp": "2026-10-17T08:20:15.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.8573675040431266, "accuracy_std_dev": 0.08566512385407106, "run_count": 15, "total_questions": 2968, "avg_cost_per_task_usd": 0.2818780539083558, "avg_duration_per_task_ms": 484741.4484501348, "avg_output_tokens_per_task": 65941.04043126684, "last_run_timestamp": "2026-10-17T08:31:19.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8571427142857143, "accuracy_std_dev": 0.022648008400073402, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15408665367965368, "avg_duration_per_task_ms": 1261266.8744588743, "avg_output_tokens_per_task": 52164.505772005774, "last_run_timestamp": "2026-10-17T16:03:34.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8571427108013937, "accuracy_std_dev": 0.01011902539224449, "run_count": 3, "total_questions": 574, "avg_cost_per_task_usd": 0.11599589721254357, "avg_duration_per_task_ms": 363022.99128919863, "avg_output_tokens_per_task": 29391.742160278747, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8570973081339713, "accuracy_std_dev": 0.00813419198867309, "run_count": 16, "total_questions": 3135, "avg_cost_per_task_usd": 0.0628228870813397, "avg_duration_per_task_ms": 581826.4315789473, "avg_output_tokens_per_task": 34631.35693779904, "last_run_timestamp": "2026-10-17T16:03:42.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8569022, "accuracy_std_dev": 0.028089651189717544, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.8333996262626261, "avg_duration_per_task_ms": 432174.9904040404, "avg_output_tokens_per_task": 33238.25858585859, "last_run_timestamp": "2026-10-17T08:18:07.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.856643153846154, "accuracy_std_dev": 0.017013380403246205, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.7557811247086247, "avg_duration_per_task_ms": 501748.9289044289, "avg_output_tokens_per_task": 30126.986013986014, "last_run_timestamp": "2026-10-17T08:02:36.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8564731094373864, "accuracy_std_dev": 0.036898649079361114, "run_count": 32, "total_questions": 5510, "avg_cost_per_task_usd": 0.014921334301270416, "avg_duration_per_task_ms": 403064.5290381125, "avg_output_tokens_per_task": 17553.35952813067, "last_run_timestamp": "2026-10-17T08:21:06.310Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8562900909090908, "accuracy_std_dev": 0.01747002531847331, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8390210973370065, "avg_duration_per_task_ms": 455775.94260789716, "avg_output_tokens_per_task": 33386.61432506887, "last_run_timestamp": "2026-10-17T08:18:06.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8562671637010675, "accuracy_std_dev": 0.055964205439230925, "run_count": 9, "total_questions": 1686, "avg_cost_per_task_usd": 0.010985191577698696, "avg_duration_per_task_ms": 499242.5960854092, "avg_output_tokens_per_task": 22245.52846975089, "last_run_timestamp": "2026-10-17T08:17:59.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8561369999999999, "accuracy_std_dev": 0.07519723650870061, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.007035742883379248, "avg_duration_per_task_ms": 646531.86271809, "avg_output_tokens_per_task": 28996.6391184573, "last_run_timestamp": "2026-10-17T08:26:19.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8561369999999997, "accuracy_std_dev": 0.01741226929756455, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.8047317961432507, "avg_duration_per_task_ms": 480071.0406336088, "avg_output_tokens_per_task": 30506.34618916437, "last_run_timestamp": "2026-10-17T08:02:36.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.8560605, "accuracy_std_dev": 0.017676499999999984, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04304399747474747, "avg_duration_per_task_ms": 130697.23232323233, "avg_output_tokens_per_task": 34274.84595959596, "last_run_timestamp": "2026-10-17T08:22:58.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8559402857142857, "accuracy_std_dev": 0.016118870642061436, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13142277200577202, "avg_duration_per_task_ms": 818557.549062049, "avg_output_tokens_per_task": 41821.506493506495, "last_run_timestamp": "2026-10-17T16:06:33.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8558689848156183, "accuracy_std_dev": 0.06273596781165733, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.2468844273318872, "avg_duration_per_task_ms": 1264380.503976862, "avg_output_tokens_per_task": 55086.25668835864, "last_run_timestamp": "2026-10-17T16:05:25.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8558500762766644, "accuracy_std_dev": 0.04264423738619322, "run_count": 8, "total_questions": 1547, "avg_cost_per_task_usd": 0.01680374466709761, "avg_duration_per_task_ms": 536166.6296056884, "avg_output_tokens_per_task": 41929.81641887524, "last_run_timestamp": "2026-10-17T08:26:18.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8553733300546447, "accuracy_std_dev": 0.04403852970379714, "run_count": 6, "total_questions": 915, "avg_cost_per_task_usd": 0.021436923497267762, "avg_duration_per_task_ms": 455969.31147540984, "avg_output_tokens_per_task": 35599.86010928962, "last_run_timestamp": "2026-10-17T08:19:17.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8553548880866427, "accuracy_std_dev": 0.040121466593740725, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.16188800866425992, "avg_duration_per_task_ms": 1012449.7552346571, "avg_output_tokens_per_task": 54542.67653429603, "last_run_timestamp": "2026-10-17T16:03:32.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8552632087114336, "accuracy_std_dev": 0.016551219440862706, "run_count": 7, "total_questions": 1102, "avg_cost_per_task_usd": 0.004268463702359346, "avg_duration_per_task_ms": 367204.3484573503, "avg_output_tokens_per_task": 9380.986388384756, "last_run_timestamp": "2026-10-17T08:21:41.875Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8550657272727274, "accuracy_std_dev": 0.012107062717806176, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.02919473599632691, "avg_duration_per_task_ms": 757966.051882461, "avg_output_tokens_per_task": 33302.242883379244, "last_run_timestamp": "2026-10-17T08:19:33.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8549618244274809, "accuracy_std_dev": 0.005347999999999964, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.22707390839694658, "avg_duration_per_task_ms": 373958.2061068702, "avg_output_tokens_per_task": 50795.681933842236, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8548968432122371, "accuracy_std_dev": 0.022342831477623858, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.043010272147864875, "avg_duration_per_task_ms": 646616.7437858508, "avg_output_tokens_per_task": 35899.3862332696, "last_run_timestamp": "2026-10-17T16:05:53.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8547321572398191, "accuracy_std_dev": 0.056526030735110595, "run_count": 9, "total_questions": 1768, "avg_cost_per_task_usd": 0.07859683484162898, "avg_duration_per_task_ms": 524667.8444570135, "avg_output_tokens_per_task": 21406.976244343892, "last_run_timestamp": "2026-10-17T08:31:20.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8539973475609757, "accuracy_std_dev": 0.0065049450612591365, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.054438141260162595, "avg_duration_per_task_ms": 424693.43699186994, "avg_output_tokens_per_task": 27839.288617886177, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.853815888888889, "accuracy_std_dev": 0.010182476570008156, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.1524213243546577, "avg_duration_per_task_ms": 432116.63383838383, "avg_output_tokens_per_task": 44104.40488215488, "last_run_timestamp": "2026-10-17T16:07:03.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.853535, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.028549383838383836, "avg_duration_per_task_ms": 582750.7070707071, "avg_output_tokens_per_task": 33383.742424242424, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8533156956521739, "accuracy_std_dev": 0.05116508602250153, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.1877648739569609, "avg_duration_per_task_ms": 41332.029863855954, "avg_output_tokens_per_task": 3381.6989459815545, "last_run_timestamp": "2026-10-17T08:20:36.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8532947142857142, "accuracy_std_dev": 0.03432999370277621, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015808059163059163, "avg_duration_per_task_ms": 355212.3253968254, "avg_output_tokens_per_task": 28357.400432900435, "last_run_timestamp": "2026-10-17T16:05:30.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8531986, "accuracy_std_dev": 0.008610563944365058, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00511969898989899, "avg_duration_per_task_ms": 561176.8353535354, "avg_output_tokens_per_task": 29257.441414141413, "last_run_timestamp": "2026-10-17T16:03:16.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8530642646239555, "accuracy_std_dev": 0.022329477337983097, "run_count": 4, "total_questions": 718, "avg_cost_per_task_usd": 0.02860370194986072, "avg_duration_per_task_ms": 460751.2409470752, "avg_output_tokens_per_task": 34791.85933147632, "last_run_timestamp": "2026-10-10T08:22:41.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8522727499999999, "accuracy_std_dev": 0.013435082822130278, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10542726830808083, "avg_duration_per_task_ms": 509657.5732323232, "avg_output_tokens_per_task": 30518.208333333332, "last_run_timestamp": "2026-10-17T08:18:05.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8522533600867679, "accuracy_std_dev": 0.014343520343232845, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.09758216124367318, "avg_duration_per_task_ms": 298428.6804049169, "avg_output_tokens_per_task": 29748.560375994217, "last_run_timestamp": "2026-10-17T08:18:07.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.8521062280219781, "accuracy_std_dev": 0.006055500000000047, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.005501717032967033, "avg_duration_per_task_ms": 539158.467032967, "avg_output_tokens_per_task": 15491.851648351649, "last_run_timestamp": "2026-10-17T08:17:58.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8515657312005884, "accuracy_std_dev": 0.01705133083065113, "run_count": 29, "total_questions": 5439, "avg_cost_per_task_usd": 0.16341508769994484, "avg_duration_per_task_ms": 193665.13403199118, "avg_output_tokens_per_task": 29005.72347858062, "last_run_timestamp": "2026-10-17T08:21:34.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8509335454545452, "accuracy_std_dev": 0.009587031319478457, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03928296464646465, "avg_duration_per_task_ms": 395425.2134986226, "avg_output_tokens_per_task": 39492.5404040404, "last_run_timestamp": "2026-10-17T16:06:26.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8507791959910913, "accuracy_std_dev": 0.007724931968526794, "run_count": 7, "total_questions": 1347, "avg_cost_per_task_usd": 0.09850432145508536, "avg_duration_per_task_ms": 703086.4394951744, "avg_output_tokens_per_task": 38320.09428359317, "last_run_timestamp": "2026-09-19T08:55:10.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8507279006896552, "accuracy_std_dev": 0.012037642028375229, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.03107361793103448, "avg_duration_per_task_ms": 729282.3756321839, "avg_output_tokens_per_task": 32410.26482758621, "last_run_timestamp": "2026-10-17T08:19:39.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.850575, "accuracy_std_dev": null, "run_count": 1, "total_questions": 145, "avg_cost_per_task_usd": 0.015425489655172412, "avg_duration_per_task_ms": 62324.744827586204, "avg_output_tokens_per_task": 15913.096551724138, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8505509090909091, "accuracy_std_dev": 0.01036149420125517, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09833260835629017, "avg_duration_per_task_ms": 575134.6868686868, "avg_output_tokens_per_task": 30653.659779614325, "last_run_timestamp": "2026-10-17T16:03:32.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8504434125475285, "accuracy_std_dev": 0.018109786799890826, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.0379920773130545, "avg_duration_per_task_ms": 600970.3225602028, "avg_output_tokens_per_task": 35466.75095057034, "last_run_timestamp": "2026-10-17T16:05:47.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8504087142857142, "accuracy_std_dev": 0.010549160084565759, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.012687737373737374, "avg_duration_per_task_ms": 687002.3088023087, "avg_output_tokens_per_task": 18864.385281385283, "last_run_timestamp": "2026-10-17T16:05:11.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.8501683333333333, "accuracy_std_dev": 0.09737163729181557, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11478661195286197, "avg_duration_per_task_ms": 1534934.6153198653, "avg_output_tokens_per_task": 28551.546296296296, "last_run_timestamp": "2026-10-17T16:06:04.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8499579999999999, "accuracy_std_dev": 0.02833895562648702, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07370817487373738, "avg_duration_per_task_ms": 515044.92234848486, "avg_output_tokens_per_task": 22619.678661616163, "last_run_timestamp": "2026-10-17T16:03:30.488Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8499179397894735, "accuracy_std_dev": 0.03519465908137437, "run_count": 36, "total_questions": 7125, "avg_cost_per_task_usd": 0.16504187410526316, "avg_duration_per_task_ms": 150293.6529122807, "avg_output_tokens_per_task": 15256.248701754386, "last_run_timestamp": "2026-10-17T08:25:18.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8497263442622951, "accuracy_std_dev": 0.01704, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.06129340163934426, "avg_duration_per_task_ms": 530265.6393442623, "avg_output_tokens_per_task": 25778.054644808744, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8491582000000001, "accuracy_std_dev": 0.03436330697648291, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.7065395328282827, "avg_duration_per_task_ms": 477955.9121212121, "avg_output_tokens_per_task": 28278.689393939392, "last_run_timestamp": "2026-10-17T08:18:06.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8489898, "accuracy_std_dev": 0.014438872440741338, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.020101904040404038, "avg_duration_per_task_ms": 416446.1484848485, "avg_output_tokens_per_task": 39974.569696969695, "last_run_timestamp": "2026-10-17T08:20:29.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.8489013956043955, "accuracy_std_dev": 0.0032350000000000434, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.06277839285714286, "avg_duration_per_task_ms": 182157.82417582418, "avg_output_tokens_per_task": 21243.684065934067, "last_run_timestamp": "2026-10-17T08:18:00.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8488907957153787, "accuracy_std_dev": 0.012007467883961828, "run_count": 8, "total_questions": 1307, "avg_cost_per_task_usd": 0.006448847742922722, "avg_duration_per_task_ms": 456157.4078041316, "avg_output_tokens_per_task": 15889.71155317521, "last_run_timestamp": "2026-10-17T16:03:10.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.8488849929577466, "accuracy_std_dev": 0.010126981561924332, "run_count": 3, "total_questions": 568, "avg_cost_per_task_usd": 0.1339099982394366, "avg_duration_per_task_ms": 312089.95246478874, "avg_output_tokens_per_task": 20114.045774647886, "last_run_timestamp": "2026-10-17T08:18:12.239Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.848859786319218, "accuracy_std_dev": 0.027189382347480782, "run_count": 8, "total_questions": 1535, "avg_cost_per_task_usd": 0.10762727947882736, "avg_duration_per_task_ms": 510720.01042345277, "avg_output_tokens_per_task": 33822.35960912052, "last_run_timestamp": "2026-10-17T08:23:18.493Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8486092684824902, "accuracy_std_dev": 0.00933062433006971, "run_count": 12, "total_questions": 2313, "avg_cost_per_task_usd": 0.06752393212278426, "avg_duration_per_task_ms": 201231.72676178123, "avg_output_tokens_per_task": 21002.852140077823, "last_run_timestamp": "2026-10-17T08:18:07.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8481527067003793, "accuracy_std_dev": 0.016458626327349835, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.05132803666245259, "avg_duration_per_task_ms": 276383.4787189212, "avg_output_tokens_per_task": 45526.28360724821, "last_run_timestamp": "2026-10-17T16:03:21.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8481498894762135, "accuracy_std_dev": 0.02068849935160807, "run_count": 11, "total_questions": 2081, "avg_cost_per_task_usd": 0.13852242095146566, "avg_duration_per_task_ms": 288258.8217203268, "avg_output_tokens_per_task": 27356.0004805382, "last_run_timestamp": "2026-10-17T08:31:19.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.8478604104595879, "accuracy_std_dev": 0.05225305319504382, "run_count": 7, "total_questions": 1262, "avg_cost_per_task_usd": 0.17750794611727413, "avg_duration_per_task_ms": 1641892.1576862123, "avg_output_tokens_per_task": 51877.93106180666, "last_run_timestamp": "2026-10-17T16:05:03.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8477753864168618, "accuracy_std_dev": 0.03714670591300797, "run_count": 4, "total_questions": 427, "avg_cost_per_task_usd": 0.10432981733021078, "avg_duration_per_task_ms": 293903.51990632317, "avg_output_tokens_per_task": 24012.651053864167, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8476576319514662, "accuracy_std_dev": 0.03635809655963856, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.09557547826086957, "avg_duration_per_task_ms": 1305294.970677452, "avg_output_tokens_per_task": 42689.957532861474, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8476008905764583, "accuracy_std_dev": 0.021332890049144525, "run_count": 15, "total_questions": 2897, "avg_cost_per_task_usd": 0.19357334760096653, "avg_duration_per_task_ms": 273006.10079392476, "avg_output_tokens_per_task": 43495.62892647566, "last_run_timestamp": "2026-10-17T16:06:16.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8475975236486487, "accuracy_std_dev": 0.01609637191005063, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.02014133502252253, "avg_duration_per_task_ms": 329056.98986486485, "avg_output_tokens_per_task": 40068.135698198195, "last_run_timestamp": "2026-10-17T08:23:24.777Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8472923333333333, "accuracy_std_dev": 0.01746311726340848, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12316125799663302, "avg_duration_per_task_ms": 547448.6262626263, "avg_output_tokens_per_task": 46449.42003367003, "last_run_timestamp": "2026-10-17T16:03:16.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8466919554707381, "accuracy_std_dev": 0.010384248473499426, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.007098816157760814, "avg_duration_per_task_ms": 559077.6908396947, "avg_output_tokens_per_task": 17567.215012722645, "last_run_timestamp": "2026-10-17T16:09:39.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8466410969581748, "accuracy_std_dev": 0.019128826891222965, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.06905485171102661, "avg_duration_per_task_ms": 249860.40684410647, "avg_output_tokens_per_task": 19302.24651457541, "last_run_timestamp": "2026-10-17T08:23:01.048Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8465608571428571, "accuracy_std_dev": 0.04257476003263826, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11109369480519479, "avg_duration_per_task_ms": 1258901.4877344877, "avg_output_tokens_per_task": 44202.00216450216, "last_run_timestamp": "2026-10-17T16:03:19.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8464300532319391, "accuracy_std_dev": 0.023377056934898808, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.04723669264892269, "avg_duration_per_task_ms": 616426.6958174905, "avg_output_tokens_per_task": 37996.90114068441, "last_run_timestamp": "2026-10-17T16:03:34.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8463673576923078, "accuracy_std_dev": 0.018423266495368705, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.03161102948717948, "avg_duration_per_task_ms": 529309.1025641026, "avg_output_tokens_per_task": 32981.63846153846, "last_run_timestamp": "2026-10-17T16:03:22.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8461700000000001, "accuracy_std_dev": 0.023688085037630202, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01534476388888889, "avg_duration_per_task_ms": 558635.6464646464, "avg_output_tokens_per_task": 37681.07765151515, "last_run_timestamp": "2026-10-17T08:04:11.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8459081790540539, "accuracy_std_dev": 0.02890872574154731, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.013105496621621622, "avg_duration_per_task_ms": 598888.7685810811, "avg_output_tokens_per_task": 39477.123873873876, "last_run_timestamp": "2026-10-17T08:23:38.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8458659444444445, "accuracy_std_dev": 0.009899875765057874, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.14837817227833897, "avg_duration_per_task_ms": 168870.18967452302, "avg_output_tokens_per_task": 32660.4012345679, "last_run_timestamp": "2026-10-17T08:26:17.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.845748875, "accuracy_std_dev": 0.019359049895575332, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1413926414141414, "avg_duration_per_task_ms": 904128.5599747475, "avg_output_tokens_per_task": 55516.943813131315, "last_run_timestamp": "2026-10-17T16:06:34.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8453582857142857, "accuracy_std_dev": 0.00738922036703828, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14179908585858583, "avg_duration_per_task_ms": 149963.48196248195, "avg_output_tokens_per_task": 32052.386724386724, "last_run_timestamp": "2026-10-17T08:23:53.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8453046666666667, "accuracy_std_dev": 0.012280214547528612, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10640831088664424, "avg_duration_per_task_ms": 345181.29517396184, "avg_output_tokens_per_task": 30203.571829405162, "last_run_timestamp": "2026-10-17T08:18:07.794Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8452150751445088, "accuracy_std_dev": 0.013405246218831629, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.08661877906229928, "avg_duration_per_task_ms": 1008472.3018625563, "avg_output_tokens_per_task": 43824.03468208093, "last_run_timestamp": "2026-10-17T16:06:14.503Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8450553324808183, "accuracy_std_dev": 0.0035741898382710214, "run_count": 4, "total_questions": 782, "avg_cost_per_task_usd": 0.10689804731457801, "avg_duration_per_task_ms": 244684.97186700767, "avg_output_tokens_per_task": 30978.448849104858, "last_run_timestamp": "2026-10-17T08:31:10.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8449956870229007, "accuracy_std_dev": 0.02085458095209539, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.10591242111959287, "avg_duration_per_task_ms": 201111.52290076335, "avg_output_tokens_per_task": 30365.776081424938, "last_run_timestamp": "2026-10-17T08:22:58.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8445056363636364, "accuracy_std_dev": 0.014307964827470473, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.13059047199265383, "avg_duration_per_task_ms": 392574.217630854, "avg_output_tokens_per_task": 38166.979338842975, "last_run_timestamp": "2026-10-17T16:05:05.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8444203561384532, "accuracy_std_dev": 0.09907049002682446, "run_count": 19, "total_questions": 3698, "avg_cost_per_task_usd": 0.008343435640886965, "avg_duration_per_task_ms": 607566.9085992428, "avg_output_tokens_per_task": 27823.074364521362, "last_run_timestamp": "2026-10-17T08:31:21.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.844398036338225, "accuracy_std_dev": 0.011789574159373825, "run_count": 8, "total_questions": 1431, "avg_cost_per_task_usd": 0.015440737246680642, "avg_duration_per_task_ms": 650403.249475891, "avg_output_tokens_per_task": 16520.188679245282, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8442990780261495, "accuracy_std_dev": 0.014090709122552674, "run_count": 12, "total_questions": 2371, "avg_cost_per_task_usd": 0.08855882834247152, "avg_duration_per_task_ms": 405312.7507380852, "avg_output_tokens_per_task": 22020.992408266553, "last_run_timestamp": "2026-10-17T16:03:21.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8442945100174215, "accuracy_std_dev": 0.015586349631435004, "run_count": 12, "total_questions": 2296, "avg_cost_per_task_usd": 0.02107914808362369, "avg_duration_per_task_ms": 263998.6659407666, "avg_output_tokens_per_task": 37097.8218641115, "last_run_timestamp": "2026-10-17T08:22:57.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8440790973451325, "accuracy_std_dev": 0.04414326163627795, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.16686697387273494, "avg_duration_per_task_ms": 164309.61483354404, "avg_output_tokens_per_task": 16467.685208596715, "last_run_timestamp": "2026-10-17T08:02:36.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8440043168316831, "accuracy_std_dev": 0.006860781750965057, "run_count": 8, "total_questions": 1515, "avg_cost_per_task_usd": 0.05332531353135314, "avg_duration_per_task_ms": 266581.80264026404, "avg_output_tokens_per_task": 22128.052145214522, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8434823631462101, "accuracy_std_dev": 0.011605077252105817, "run_count": 16, "total_questions": 3153, "avg_cost_per_task_usd": 0.11780266603235014, "avg_duration_per_task_ms": 324853.04471931496, "avg_output_tokens_per_task": 34347.03774183318, "last_run_timestamp": "2026-10-17T08:23:25.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8433546514886163, "accuracy_std_dev": 0.019866857652736186, "run_count": 26, "total_questions": 5139, "avg_cost_per_task_usd": 0.24385491282350655, "avg_duration_per_task_ms": 503356.83965752093, "avg_output_tokens_per_task": 47257.87332165791, "last_run_timestamp": "2026-10-17T16:06:05.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8432919898534386, "accuracy_std_dev": 0.016413678415754425, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.009925733934611049, "avg_duration_per_task_ms": 452405.15670800454, "avg_output_tokens_per_task": 31483.80439684329, "last_run_timestamp": "2026-10-17T08:31:20.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8430333386243387, "accuracy_std_dev": 0.009092680280725797, "run_count": 8, "total_questions": 1323, "avg_cost_per_task_usd": 0.13841274300831444, "avg_duration_per_task_ms": 126887.37868480726, "avg_output_tokens_per_task": 31223.573696145126, "last_run_timestamp": "2026-10-17T08:26:17.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8427329166666666, "accuracy_std_dev": 0.033483254955420955, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05271298779461279, "avg_duration_per_task_ms": 212758.7373737374, "avg_output_tokens_per_task": 21862.314814814814, "last_run_timestamp": "2026-10-17T08:20:51.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8421298152723189, "accuracy_std_dev": 0.012413964315426865, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.13897624312184165, "avg_duration_per_task_ms": 200873.80123526108, "avg_output_tokens_per_task": 31356.491297024142, "last_run_timestamp": "2026-10-17T08:26:31.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8419978532110092, "accuracy_std_dev": 0.010540078834619807, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.027336418960244645, "avg_duration_per_task_ms": 516242.43323139654, "avg_output_tokens_per_task": 22594.290519877675, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8419939695817491, "accuracy_std_dev": 0.05773343437894112, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.08744185044359949, "avg_duration_per_task_ms": 697734.2648922687, "avg_output_tokens_per_task": 26121.35234474018, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8416337193675889, "accuracy_std_dev": 0.027997748302318905, "run_count": 7, "total_questions": 1265, "avg_cost_per_task_usd": 0.09091536363636364, "avg_duration_per_task_ms": 1534224.6964426877, "avg_output_tokens_per_task": 41867.99051383399, "last_run_timestamp": "2026-10-17T16:06:12.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8413977096774194, "accuracy_std_dev": 0.007842951293996439, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.03688066532258065, "avg_duration_per_task_ms": 47685.283602150535, "avg_output_tokens_per_task": 14664.099462365592, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8411896666666667, "accuracy_std_dev": 0.008729468916008341, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.036658867003367, "avg_duration_per_task_ms": 51092.77104377104, "avg_output_tokens_per_task": 14574.808080808081, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8409475076142132, "accuracy_std_dev": 0.005443376341940744, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.020375204737732657, "avg_duration_per_task_ms": 653417.3384094755, "avg_output_tokens_per_task": 23451.566835871403, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.8406283333333334, "accuracy_std_dev": 0.014825937482069168, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2733430589225589, "avg_duration_per_task_ms": 397861.94276094274, "avg_output_tokens_per_task": 61869.08417508417, "last_run_timestamp": "2026-10-17T16:03:34.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8405724, "accuracy_std_dev": 0.051235218214037107, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1625054691919192, "avg_duration_per_task_ms": 161406.5494949495, "avg_output_tokens_per_task": 16082.878282828282, "last_run_timestamp": "2026-10-17T08:20:51.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.84048825, "accuracy_std_dev": 0.004667830993887862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06197197222222223, "avg_duration_per_task_ms": 84839.70707070707, "avg_output_tokens_per_task": 13643.420454545454, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8402775531250001, "accuracy_std_dev": 0.007466846475587956, "run_count": 10, "total_questions": 1920, "avg_cost_per_task_usd": 0.039408469791666664, "avg_duration_per_task_ms": 51791.722395833334, "avg_output_tokens_per_task": 14757.431770833333, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8402425498181818, "accuracy_std_dev": 0.00976753775827742, "run_count": 8, "total_questions": 1375, "avg_cost_per_task_usd": 0.0796483170909091, "avg_duration_per_task_ms": 509723.63709090906, "avg_output_tokens_per_task": 19683.845818181817, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.8401923283835383, "accuracy_std_dev": 0.010616197274749037, "run_count": 12, "total_questions": 2357, "avg_cost_per_task_usd": 0.07210711752227408, "avg_duration_per_task_ms": 388115.4365719134, "avg_output_tokens_per_task": 21949.04836656767, "last_run_timestamp": "2026-10-17T08:18:57.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.839947, "accuracy_std_dev": 0.010969701701895635, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006264753968253967, "avg_duration_per_task_ms": 399380.0064935065, "avg_output_tokens_per_task": 25886.00432900433, "last_run_timestamp": "2026-10-17T16:05:38.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8398760924440692, "accuracy_std_dev": 0.02907933486267448, "run_count": 12, "total_questions": 2369, "avg_cost_per_task_usd": 0.01278074419586323, "avg_duration_per_task_ms": 235302.36935415788, "avg_output_tokens_per_task": 14226.744195863233, "last_run_timestamp": "2026-10-17T08:26:18.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8397788936464089, "accuracy_std_dev": 0.056010342147862845, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.0787299079189687, "avg_duration_per_task_ms": 572401.2780847145, "avg_output_tokens_per_task": 44061.1864640884, "last_run_timestamp": "2026-10-17T16:07:16.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.8395436603119584, "accuracy_std_dev": 0.010997134534013122, "run_count": 7, "total_questions": 1154, "avg_cost_per_task_usd": 0.006737056325823224, "avg_duration_per_task_ms": 622594.8396880416, "avg_output_tokens_per_task": 16640.68890814558, "last_run_timestamp": "2026-10-17T16:06:04.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8395171528268551, "accuracy_std_dev": 0.020340296193762753, "run_count": 8, "total_questions": 1132, "avg_cost_per_task_usd": 0.016312568021201415, "avg_duration_per_task_ms": 87770.31007067137, "avg_output_tokens_per_task": 13991.570671378091, "last_run_timestamp": "2026-10-17T16:05:36.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8384655804464972, "accuracy_std_dev": 0.017858184675894877, "run_count": 7, "total_questions": 1299, "avg_cost_per_task_usd": 0.15115439722863744, "avg_duration_per_task_ms": 297429.6466512702, "avg_output_tokens_per_task": 35258.00923787529, "last_run_timestamp": "2026-10-17T08:21:08.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8383907751724139, "accuracy_std_dev": 0.014934618806956231, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.22592824689655172, "avg_duration_per_task_ms": 279235.183908046, "avg_output_tokens_per_task": 51209.457931034485, "last_run_timestamp": "2026-10-17T16:05:55.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8378307503892062, "accuracy_std_dev": 0.03618934998158991, "run_count": 10, "total_questions": 1927, "avg_cost_per_task_usd": 0.029675656979761288, "avg_duration_per_task_ms": 291090.3741567203, "avg_output_tokens_per_task": 9165.982355993772, "last_run_timestamp": "2026-10-17T08:18:00.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8378226666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.15828378956228956, "avg_duration_per_task_ms": 446608.7861952862, "avg_output_tokens_per_task": 56298.93097643097, "last_run_timestamp": "2026-10-10T08:30:28.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8378130592547341, "accuracy_std_dev": 0.02159011628523284, "run_count": 9, "total_questions": 1637, "avg_cost_per_task_usd": 0.07307316065974342, "avg_duration_per_task_ms": 388815.5760537569, "avg_output_tokens_per_task": 24094.505803298718, "last_run_timestamp": "2026-10-17T16:05:55.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8371951481707317, "accuracy_std_dev": 0.02877358996641808, "run_count": 9, "total_questions": 1640, "avg_cost_per_task_usd": 0.011173640853658538, "avg_duration_per_task_ms": 373621.1006097561, "avg_output_tokens_per_task": 22130.590853658538, "last_run_timestamp": "2026-10-17T08:20:32.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.836700375, "accuracy_std_dev": 0.05226306970973264, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09356622285353536, "avg_duration_per_task_ms": 408622.6205808081, "avg_output_tokens_per_task": 21042.78724747475, "last_run_timestamp": "2026-10-17T16:03:34.731Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.836653454183267, "accuracy_std_dev": 0.007833999999999952, "run_count": 2, "total_questions": 251, "avg_cost_per_task_usd": 0.06977503187250997, "avg_duration_per_task_ms": 237256.35059760956, "avg_output_tokens_per_task": 17511.450199203187, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8365635357637288, "accuracy_std_dev": 0.030013675310646196, "run_count": 11, "total_questions": 2167, "avg_cost_per_task_usd": 0.06634258514074758, "avg_duration_per_task_ms": 1054714.62113521, "avg_output_tokens_per_task": 33093.440239963085, "last_run_timestamp": "2026-10-17T16:03:24.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8364136921606119, "accuracy_std_dev": 0.02209302746873717, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.09194236265137029, "avg_duration_per_task_ms": 1028125.6876991715, "avg_output_tokens_per_task": 42157.6908859146, "last_run_timestamp": "2026-10-17T16:07:08.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8364074347826086, "accuracy_std_dev": 0.0120242022634283, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.26040066996047434, "avg_duration_per_task_ms": 271238.8100570927, "avg_output_tokens_per_task": 56706.782608695656, "last_run_timestamp": "2026-10-17T16:06:15.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.836088, "accuracy_std_dev": 0.012542317105050277, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.08354139898989901, "avg_duration_per_task_ms": 748339.285123967, "avg_output_tokens_per_task": 48591.798898071625, "last_run_timestamp": "2026-10-17T16:06:53.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.836088, "accuracy_std_dev": 0.011196343721379466, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.24613506887052342, "avg_duration_per_task_ms": 366252.56749311293, "avg_output_tokens_per_task": 39848.296143250685, "last_run_timestamp": "2026-10-17T08:20:12.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8360502124789209, "accuracy_std_dev": 0.025222744943188383, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.15836738055087127, "avg_duration_per_task_ms": 914252.053400787, "avg_output_tokens_per_task": 52022.07419898819, "last_run_timestamp": "2026-10-17T16:04:06.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8358585, "accuracy_std_dev": 0.014603658180401272, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.15205888257575756, "avg_duration_per_task_ms": 235007.2601010101, "avg_output_tokens_per_task": 34304.49873737374, "last_run_timestamp": "2026-10-17T08:23:37.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8357648888888889, "accuracy_std_dev": 0.023650418015776994, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11090620426487094, "avg_duration_per_task_ms": 397711.55667789, "avg_output_tokens_per_task": 36242.744668911335, "last_run_timestamp": "2026-10-17T08:23:46.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8357119447513812, "accuracy_std_dev": 0.030735575803944197, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.01723706399631676, "avg_duration_per_task_ms": 153182.64871086556, "avg_output_tokens_per_task": 18067.730662983424, "last_run_timestamp": "2026-10-17T16:05:57.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8356714444444444, "accuracy_std_dev": 0.013858410451668465, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14028259147025812, "avg_duration_per_task_ms": 183217.68013468012, "avg_output_tokens_per_task": 31651.560044893376, "last_run_timestamp": "2026-10-17T08:23:38.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8356163613013698, "accuracy_std_dev": 0.008544206082681626, "run_count": 6, "total_questions": 1168, "avg_cost_per_task_usd": 0.02781799058219178, "avg_duration_per_task_ms": 605091.2380136986, "avg_output_tokens_per_task": 14771.433219178081, "last_run_timestamp": "2026-10-17T08:19:50.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8350166666666667, "accuracy_std_dev": 0.005991526757754575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06194655555555556, "avg_duration_per_task_ms": 84396.87373737374, "avg_output_tokens_per_task": 13638.5, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8345362173215719, "accuracy_std_dev": 0.009596989300434438, "run_count": 7, "total_questions": 1247, "avg_cost_per_task_usd": 0.045831162790697676, "avg_duration_per_task_ms": 89512.38492381717, "avg_output_tokens_per_task": 15115.87570168404, "last_run_timestamp": "2026-10-17T08:23:46.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8343871437605396, "accuracy_std_dev": 0.02214593854208312, "run_count": 12, "total_questions": 2372, "avg_cost_per_task_usd": 0.024504262225969645, "avg_duration_per_task_ms": 383677.22807757166, "avg_output_tokens_per_task": 45255.54173693086, "last_run_timestamp": "2026-10-17T08:18:07.356Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8340346666666668, "accuracy_std_dev": 0.033680117491217625, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05396650210437711, "avg_duration_per_task_ms": 255758.39562289562, "avg_output_tokens_per_task": 22970.753367003366, "last_run_timestamp": "2026-10-17T08:02:36.130Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8340288847157016, "accuracy_std_dev": 0.01861503359435616, "run_count": 11, "total_questions": 1917, "avg_cost_per_task_usd": 0.09581860772039645, "avg_duration_per_task_ms": 622289.2008346375, "avg_output_tokens_per_task": 29867.270213875847, "last_run_timestamp": "2026-10-17T16:03:34.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8337731531353135, "accuracy_std_dev": 0.013206984544172061, "run_count": 8, "total_questions": 1515, "avg_cost_per_task_usd": 0.22087456369636965, "avg_duration_per_task_ms": 445733.97095709573, "avg_output_tokens_per_task": 50080.22772277228, "last_run_timestamp": "2026-10-17T16:03:34.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.83367, "accuracy_std_dev": 0.01234350860169023, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0391534101010101, "avg_duration_per_task_ms": 225964.9994949495, "avg_output_tokens_per_task": 14531.921212121211, "last_run_timestamp": "2026-10-17T08:02:36.310Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8336229634994206, "accuracy_std_dev": 0.021738738072566568, "run_count": 9, "total_questions": 1726, "avg_cost_per_task_usd": 0.020348153534183084, "avg_duration_per_task_ms": 236255.9015063731, "avg_output_tokens_per_task": 40476.10834298957, "last_run_timestamp": "2026-10-17T08:31:21.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8334862727272728, "accuracy_std_dev": 0.06978429003083315, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.20374753076216714, "avg_duration_per_task_ms": 537318.2309458219, "avg_output_tokens_per_task": 46322.687786960516, "last_run_timestamp": "2026-10-17T16:06:55.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.8333332476370511, "accuracy_std_dev": 0.03496673477303106, "run_count": 4, "total_questions": 529, "avg_cost_per_task_usd": 0.09262325141776938, "avg_duration_per_task_ms": 324581.95463138, "avg_output_tokens_per_task": 38346.877126654064, "last_run_timestamp": "2026-10-17T16:03:14.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8332576446056212, "accuracy_std_dev": 0.012460556042274078, "run_count": 14, "total_questions": 2206, "avg_cost_per_task_usd": 0.11502464823209428, "avg_duration_per_task_ms": 224648.71078875792, "avg_output_tokens_per_task": 27449.897552130555, "last_run_timestamp": "2026-10-17T08:03:37.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8328165280620154, "accuracy_std_dev": 0.017273176086970456, "run_count": 17, "total_questions": 3225, "avg_cost_per_task_usd": 0.23915214232558138, "avg_duration_per_task_ms": 374125.24062015506, "avg_output_tokens_per_task": 42949.212093023256, "last_run_timestamp": "2026-10-17T16:06:00.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.832801601148692, "accuracy_std_dev": 0.01796862582217962, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.015661317804722397, "avg_duration_per_task_ms": 438016.0848755584, "avg_output_tokens_per_task": 40296.81493299298, "last_run_timestamp": "2026-10-17T08:23:38.619Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.8327431107738998, "accuracy_std_dev": 0.03389053866730361, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.10782147395042994, "avg_duration_per_task_ms": 1015295.3773394031, "avg_output_tokens_per_task": 33688.65048052605, "last_run_timestamp": "2026-10-17T16:03:19.092Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8321570888335409, "accuracy_std_dev": 0.0106410052060675, "run_count": 15, "total_questions": 2409, "avg_cost_per_task_usd": 0.01938649107513491, "avg_duration_per_task_ms": 213034.18430884185, "avg_output_tokens_per_task": 38582.406392694065, "last_run_timestamp": "2026-10-17T08:20:36.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8320707500000001, "accuracy_std_dev": 0.008392927678756304, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.08125586531986531, "avg_duration_per_task_ms": 697034.4953703703, "avg_output_tokens_per_task": 56943.53914141414, "last_run_timestamp": "2026-10-17T16:07:34.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8319781768292682, "accuracy_std_dev": 0.02156774450887252, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.07742283333333333, "avg_duration_per_task_ms": 694188.4644308944, "avg_output_tokens_per_task": 41828.17479674797, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.83165, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009051732323232323, "avg_duration_per_task_ms": 428104.83838383836, "avg_output_tokens_per_task": 21594.323232323233, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.8316496666666667, "accuracy_std_dev": 0.009013484577133449, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02192354882154882, "avg_duration_per_task_ms": 12405.745791245792, "avg_output_tokens_per_task": 14165.61616161616, "last_run_timestamp": "2026-10-17T08:04:11.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8315218454968942, "accuracy_std_dev": 0.015272735801327252, "run_count": 7, "total_questions": 1288, "avg_cost_per_task_usd": 0.022463093167701866, "avg_duration_per_task_ms": 302352.67624223605, "avg_output_tokens_per_task": 44704.51242236025, "last_run_timestamp": "2026-10-17T08:21:10.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.831313, "accuracy_std_dev": 0.01944119231940264, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025460874747474747, "avg_duration_per_task_ms": 242198.30808080808, "avg_output_tokens_per_task": 21221.607070707072, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8310885000000001, "accuracy_std_dev": 0.012955632388399007, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06418621717171719, "avg_duration_per_task_ms": 117116.24831649831, "avg_output_tokens_per_task": 13483.090067340067, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8308479888164026, "accuracy_std_dev": 0.015453198666473883, "run_count": 14, "total_questions": 2146, "avg_cost_per_task_usd": 0.037517430102516305, "avg_duration_per_task_ms": 263231.2437092265, "avg_output_tokens_per_task": 29907.408201304752, "last_run_timestamp": "2026-10-17T16:03:24.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8307303478466634, "accuracy_std_dev": 0.010455075072751015, "run_count": 11, "total_questions": 2113, "avg_cost_per_task_usd": 0.22723341410317086, "avg_duration_per_task_ms": 294845.04164694744, "avg_output_tokens_per_task": 51477.59678182679, "last_run_timestamp": "2026-10-17T16:05:16.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8305271666666666, "accuracy_std_dev": 0.004924217515391559, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05513053872053872, "avg_duration_per_task_ms": 198190.63888888888, "avg_output_tokens_per_task": 15260.853535353535, "last_run_timestamp": "2026-10-17T08:22:58.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8304393877813504, "accuracy_std_dev": 0.01344638632449253, "run_count": 8, "total_questions": 1555, "avg_cost_per_task_usd": 0.1521527459807074, "avg_duration_per_task_ms": 258681.17106109325, "avg_output_tokens_per_task": 34390.49389067524, "last_run_timestamp": "2026-10-17T08:31:20.051Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8302596014344261, "accuracy_std_dev": 0.00648568653960398, "run_count": 10, "total_questions": 1952, "avg_cost_per_task_usd": 0.03510589190573771, "avg_duration_per_task_ms": 205657.60348360657, "avg_output_tokens_per_task": 14193.383709016394, "last_run_timestamp": "2026-10-17T08:30:44.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8286928213457077, "accuracy_std_dev": 0.0309478649146514, "run_count": 7, "total_questions": 1293, "avg_cost_per_task_usd": 0.006722985305491105, "avg_duration_per_task_ms": 672885.3240525909, "avg_output_tokens_per_task": 18073.616395978344, "last_run_timestamp": "2026-10-17T16:03:30.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.8283759292307693, "accuracy_std_dev": 0.023115878020096917, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.0696730758974359, "avg_duration_per_task_ms": 827348.3805128206, "avg_output_tokens_per_task": 39830.013333333336, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.82828275, "accuracy_std_dev": 0.004610363346581274, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0223316047979798, "avg_duration_per_task_ms": 615131.3409090909, "avg_output_tokens_per_task": 56159.15404040404, "last_run_timestamp": "2026-10-17T08:18:07.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8282827272727273, "accuracy_std_dev": 0.019724543474799548, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.2436875546372819, "avg_duration_per_task_ms": 366201.29155188246, "avg_output_tokens_per_task": 55218.82966023875, "last_run_timestamp": "2026-10-17T16:06:12.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8279999858823529, "accuracy_std_dev": 0.022369675456957054, "run_count": 11, "total_questions": 2125, "avg_cost_per_task_usd": 0.02507603670588236, "avg_duration_per_task_ms": 878373.8503529412, "avg_output_tokens_per_task": 46965.459294117645, "last_run_timestamp": "2026-10-17T08:23:38.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8278469506242907, "accuracy_std_dev": 0.008508360090646084, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.007694182179341657, "avg_duration_per_task_ms": 337110.9665153235, "avg_output_tokens_per_task": 19055.32803632236, "last_run_timestamp": "2026-10-17T16:06:33.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8276515833333333, "accuracy_std_dev": 0.024461777935036868, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.038061598905723915, "avg_duration_per_task_ms": 345575.1603535354, "avg_output_tokens_per_task": 39167.65488215488, "last_run_timestamp": "2026-10-17T16:06:09.632Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8274411249999999, "accuracy_std_dev": 0.01202255100256912, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03823982765151515, "avg_duration_per_task_ms": 264254.53851010103, "avg_output_tokens_per_task": 15517.1773989899, "last_run_timestamp": "2026-10-17T08:18:05.671Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8273644545454546, "accuracy_std_dev": 0.02235000962929887, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.13616948393021117, "avg_duration_per_task_ms": 546895.1822773187, "avg_output_tokens_per_task": 43108.64462809917, "last_run_timestamp": "2026-10-17T16:03:32.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8273474444444444, "accuracy_std_dev": 0.006968947666631396, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13947370089786754, "avg_duration_per_task_ms": 169790.02132435466, "avg_output_tokens_per_task": 34460.64870931538, "last_run_timestamp": "2026-10-17T08:18:00.675Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8273257868177137, "accuracy_std_dev": 0.013461365335284555, "run_count": 10, "total_questions": 1942, "avg_cost_per_task_usd": 0.0351326035015448, "avg_duration_per_task_ms": 424285.9001029866, "avg_output_tokens_per_task": 48592.330072090626, "last_run_timestamp": "2026-10-17T08:18:01.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8272847449686571, "accuracy_std_dev": 0.021180314014064436, "run_count": 16, "total_questions": 3031, "avg_cost_per_task_usd": 0.04213694556252062, "avg_duration_per_task_ms": 414089.77202243486, "avg_output_tokens_per_task": 45769.54503464203, "last_run_timestamp": "2026-10-17T16:03:24.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8271604444444445, "accuracy_std_dev": 0.023434861676623345, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0269165101010101, "avg_duration_per_task_ms": 568738.2138047138, "avg_output_tokens_per_task": 46021.95230078563, "last_run_timestamp": "2026-10-17T08:17:59.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8271603333333334, "accuracy_std_dev": 0.018009865870313054, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.12849964365881034, "avg_duration_per_task_ms": 324347.8726150393, "avg_output_tokens_per_task": 53019.92480359147, "last_run_timestamp": "2026-10-17T16:06:38.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.826809625, "accuracy_std_dev": 0.013025065527066464, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09912519444444444, "avg_duration_per_task_ms": 360768.5435606061, "avg_output_tokens_per_task": 45556.78724747475, "last_run_timestamp": "2026-10-17T16:06:16.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.8265989999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009182863636363637, "avg_duration_per_task_ms": 852550.7929292929, "avg_output_tokens_per_task": 21914.838383838385, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8264122222222224, "accuracy_std_dev": 0.010177010309491996, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04327315263748596, "avg_duration_per_task_ms": 229752.91245791246, "avg_output_tokens_per_task": 14308.176767676769, "last_run_timestamp": "2026-10-17T08:20:04.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8262167272727274, "accuracy_std_dev": 0.05836834255140663, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.05152247704315887, "avg_duration_per_task_ms": 533182.7640036731, "avg_output_tokens_per_task": 24667.44031221304, "last_run_timestamp": "2026-10-17T08:02:36.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8261783333333332, "accuracy_std_dev": 0.014287137159774959, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.06237826599326599, "avg_duration_per_task_ms": 538208.1666666666, "avg_output_tokens_per_task": 49120.39562289562, "last_run_timestamp": "2026-10-17T16:07:11.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8258873707224335, "accuracy_std_dev": 0.009676826643527784, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.025350776615969583, "avg_duration_per_task_ms": 120122.08840304182, "avg_output_tokens_per_task": 20151.38783269962, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.8254771666666667, "accuracy_std_dev": 0.00940686748102449, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.14361566498316497, "avg_duration_per_task_ms": 426029.37457912456, "avg_output_tokens_per_task": 28741.069865319867, "last_run_timestamp": "2026-10-17T08:21:06.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8254611153846155, "accuracy_std_dev": 0.015660417265753186, "run_count": 8, "total_questions": 1482, "avg_cost_per_task_usd": 0.024416639676113364, "avg_duration_per_task_ms": 282189.7125506073, "avg_output_tokens_per_task": 48622.69095816464, "last_run_timestamp": "2026-10-17T08:18:07.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8249891001283697, "accuracy_std_dev": 0.05097064513194498, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.09995508172871204, "avg_duration_per_task_ms": 541954.0753102268, "avg_output_tokens_per_task": 29446.99828840394, "last_run_timestamp": "2026-10-18T01:36:16.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8249519000000001, "accuracy_std_dev": 0.01363772648859694, "run_count": 9, "total_questions": 1730, "avg_cost_per_task_usd": 0.024441546242774562, "avg_duration_per_task_ms": 449419.536416185, "avg_output_tokens_per_task": 24400.59768786127, "last_run_timestamp": "2026-10-17T16:07:00.439Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.824916, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022821656565656566, "avg_duration_per_task_ms": 516406.73232323234, "avg_output_tokens_per_task": 45403.40909090909, "last_run_timestamp": "2026-10-04T03:04:38.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8245299743589743, "accuracy_std_dev": 0.015168242439056692, "run_count": 10, "total_questions": 1950, "avg_cost_per_task_usd": 0.1439138517948718, "avg_duration_per_task_ms": 294902.17794871796, "avg_output_tokens_per_task": 32491.769743589743, "last_run_timestamp": "2026-10-17T08:19:50.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8244948000000002, "accuracy_std_dev": 0.0601779527199123, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.028505976262626263, "avg_duration_per_task_ms": 700158.9757575757, "avg_output_tokens_per_task": 29700.529292929292, "last_run_timestamp": "2026-10-17T08:18:04.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8239975454545455, "accuracy_std_dev": 0.022862830634275574, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.010785916437098255, "avg_duration_per_task_ms": 717852.0280073462, "avg_output_tokens_per_task": 38282.08126721763, "last_run_timestamp": "2026-10-17T08:02:36.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8239056, "accuracy_std_dev": 0.013153034579138006, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02469372222222222, "avg_duration_per_task_ms": 472106.2909090909, "avg_output_tokens_per_task": 23003.946464646466, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8237281106367841, "accuracy_std_dev": 0.017561497390472025, "run_count": 15, "total_questions": 2811, "avg_cost_per_task_usd": 0.03114107612949128, "avg_duration_per_task_ms": 263866.4489505514, "avg_output_tokens_per_task": 14017.920313055853, "last_run_timestamp": "2026-10-17T08:18:05.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.8236842684210525, "accuracy_std_dev": 0.00005100000000002325, "run_count": 2, "total_questions": 380, "avg_cost_per_task_usd": 0.11858948157894737, "avg_duration_per_task_ms": 197797.2657894737, "avg_output_tokens_per_task": 29495, "last_run_timestamp": "2026-10-17T08:18:07.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8236161787072243, "accuracy_std_dev": 0.007591822309852583, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.007953439797211661, "avg_duration_per_task_ms": 483107.97084917617, "avg_output_tokens_per_task": 20467.734474017743, "last_run_timestamp": "2026-10-17T16:03:30.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8235687999999998, "accuracy_std_dev": 0.003085731900214278, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.018559439393939392, "avg_duration_per_task_ms": 535812.6131313131, "avg_output_tokens_per_task": 18445.026262626263, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8234428749999999, "accuracy_std_dev": 0.008223286849513092, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04775078093434344, "avg_duration_per_task_ms": 439035.39835858584, "avg_output_tokens_per_task": 26449.10542929293, "last_run_timestamp": "2026-10-17T16:06:45.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.823296396946565, "accuracy_std_dev": 0.009206241083574164, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.023398153519932147, "avg_duration_per_task_ms": 341450.7421543681, "avg_output_tokens_per_task": 20864.225614927906, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8232322999999999, "accuracy_std_dev": 0.020878107083976735, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.03450791515151515, "avg_duration_per_task_ms": 265271.5378787879, "avg_output_tokens_per_task": 14464.131818181819, "last_run_timestamp": "2026-10-17T08:19:16.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025912994949494946, "avg_duration_per_task_ms": 162312.898989899, "avg_output_tokens_per_task": 9036.257575757576, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8223551976047904, "accuracy_std_dev": 0.029631373343505716, "run_count": 6, "total_questions": 1169, "avg_cost_per_task_usd": 0.06724194183062446, "avg_duration_per_task_ms": 236148.74337040205, "avg_output_tokens_per_task": 20314.131736526946, "last_run_timestamp": "2026-10-17T08:26:17.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8223140000000001, "accuracy_std_dev": 0.020822165248512558, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.012249293847566574, "avg_duration_per_task_ms": 720758.7061524334, "avg_output_tokens_per_task": 36235.862258953166, "last_run_timestamp": "2026-10-17T08:04:21.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.8222643415559772, "accuracy_std_dev": 0.014072964998868393, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.1947691688804554, "avg_duration_per_task_ms": 969270.2144212524, "avg_output_tokens_per_task": 51840.38583175206, "last_run_timestamp": "2026-10-17T16:06:43.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.82211, "accuracy_std_dev": 0.057194952006274134, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09893642424242423, "avg_duration_per_task_ms": 769490.925925926, "avg_output_tokens_per_task": 39542.70622895623, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8220689337931035, "accuracy_std_dev": 0.04423033551106625, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.085449148045977, "avg_duration_per_task_ms": 780476.7949425287, "avg_output_tokens_per_task": 31176.493793103447, "last_run_timestamp": "2026-10-17T16:03:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.821895294117647, "accuracy_std_dev": 0.07409118108715027, "run_count": 3, "total_questions": 408, "avg_cost_per_task_usd": 0.13934752450980392, "avg_duration_per_task_ms": 1455873.1495098039, "avg_output_tokens_per_task": 48235.80147058824, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8214198161925602, "accuracy_std_dev": 0.04954945097072217, "run_count": 8, "total_questions": 1371, "avg_cost_per_task_usd": 0.04645164551422319, "avg_duration_per_task_ms": 608936.6484318016, "avg_output_tokens_per_task": 28638.388767323122, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8210063161094224, "accuracy_std_dev": 0.031028311643400763, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.025833400202634246, "avg_duration_per_task_ms": 497222.336372847, "avg_output_tokens_per_task": 24022.691995947316, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.8203461428571428, "accuracy_std_dev": 0.030285949285476416, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09649056854256852, "avg_duration_per_task_ms": 1458719.1861471862, "avg_output_tokens_per_task": 44637.40115440115, "last_run_timestamp": "2026-10-17T16:06:17.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8201458333333331, "accuracy_std_dev": 0.013691135080976862, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.314472340909091, "avg_duration_per_task_ms": 289534.5925925926, "avg_output_tokens_per_task": 20775.11153198653, "last_run_timestamp": "2026-10-17T08:04:11.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8197122727272727, "accuracy_std_dev": 0.018601160931956084, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.042628178145087244, "avg_duration_per_task_ms": 234970.43434343435, "avg_output_tokens_per_task": 14213.333333333334, "last_run_timestamp": "2026-10-17T08:22:58.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8192149207920791, "accuracy_std_dev": 0.018823182524040232, "run_count": 6, "total_questions": 909, "avg_cost_per_task_usd": 0.11224348184818483, "avg_duration_per_task_ms": 401079.6094609461, "avg_output_tokens_per_task": 46532.80198019802, "last_run_timestamp": "2026-10-17T16:06:46.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8191437142857143, "accuracy_std_dev": 0.03986796895724072, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04288789971139971, "avg_duration_per_task_ms": 992008.3816738817, "avg_output_tokens_per_task": 26956.146464646463, "last_run_timestamp": "2026-10-17T08:20:47.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.8188131249999999, "accuracy_std_dev": 0.0780194627391741, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01174183143939394, "avg_duration_per_task_ms": 760343.6117424242, "avg_output_tokens_per_task": 25978.727272727272, "last_run_timestamp": "2026-10-17T16:11:29.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8187420045395589, "accuracy_std_dev": 0.05002500478245231, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.02404102204928664, "avg_duration_per_task_ms": 913283.17769131, "avg_output_tokens_per_task": 46875.09792477302, "last_run_timestamp": "2026-10-17T08:26:17.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.8186202474674383, "accuracy_std_dev": 0.008502623002248598, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.03867190303907381, "avg_duration_per_task_ms": 207333.16425470333, "avg_output_tokens_per_task": 14772.572358900145, "last_run_timestamp": "2026-10-17T08:18:02.516Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.8185119845094664, "accuracy_std_dev": 0.016376070492197817, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.018653541594951233, "avg_duration_per_task_ms": 606686.3998852553, "avg_output_tokens_per_task": 20040.86230636833, "last_run_timestamp": "2026-10-17T16:06:52.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8183687777777777, "accuracy_std_dev": 0.014898891142764636, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03439966722783389, "avg_duration_per_task_ms": 222983.87822671156, "avg_output_tokens_per_task": 13813.295735129068, "last_run_timestamp": "2026-10-17T08:18:01.964Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8181818, "accuracy_std_dev": 0.02137456277353997, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1023312393939394, "avg_duration_per_task_ms": 562383.997979798, "avg_output_tokens_per_task": 37995.12222222222, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8177376144306652, "accuracy_std_dev": 0.03151916867911054, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.013423886696730551, "avg_duration_per_task_ms": 855848.4718151072, "avg_output_tokens_per_task": 39679.556933483655, "last_run_timestamp": "2026-10-17T08:19:03.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.8175460387794024, "accuracy_std_dev": 0.054722959283415716, "run_count": 8, "total_questions": 1573, "avg_cost_per_task_usd": 0.1901260775588048, "avg_duration_per_task_ms": 615386.0864589956, "avg_output_tokens_per_task": 45248.49332485696, "last_run_timestamp": "2026-10-17T16:04:35.937Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8174118475609755, "accuracy_std_dev": 0.019551780804826967, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.02269767581300813, "avg_duration_per_task_ms": 492334.5833333333, "avg_output_tokens_per_task": 22647.941056910568, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.8173077377622378, "accuracy_std_dev": 0.04183386114142466, "run_count": 3, "total_questions": 572, "avg_cost_per_task_usd": 0.00918055944055944, "avg_duration_per_task_ms": 556054.2482517483, "avg_output_tokens_per_task": 20397.216783216783, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8168566996197718, "accuracy_std_dev": 0.004401271634425658, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.009628157160963245, "avg_duration_per_task_ms": 739454.9480354879, "avg_output_tokens_per_task": 18256.190114068442, "last_run_timestamp": "2026-10-17T16:06:46.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8165709598662209, "accuracy_std_dev": 0.061324306686151714, "run_count": 22, "total_questions": 4186, "avg_cost_per_task_usd": 0.0074439737219302434, "avg_duration_per_task_ms": 498928.61681796465, "avg_output_tokens_per_task": 17412.904204491162, "last_run_timestamp": "2026-10-17T16:06:04.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.81649825, "accuracy_std_dev": 0.019083748339555836, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006335979166666667, "avg_duration_per_task_ms": 484646.10353535356, "avg_output_tokens_per_task": 18725.272727272728, "last_run_timestamp": "2026-10-17T16:03:10.763Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.81607725, "accuracy_std_dev": 0.00955115719101616, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03420622222222222, "avg_duration_per_task_ms": 636556.8263888889, "avg_output_tokens_per_task": 16058.70707070707, "last_run_timestamp": "2026-10-17T08:19:36.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8160686215384615, "accuracy_std_dev": 0.03359599240742859, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.026009656410256408, "avg_duration_per_task_ms": 463494.8851282051, "avg_output_tokens_per_task": 21626.916923076922, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8159994814814814, "accuracy_std_dev": 0.01147137633051603, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.3335022566404788, "avg_duration_per_task_ms": 298311.0364758698, "avg_output_tokens_per_task": 21104.580621025067, "last_run_timestamp": "2026-10-17T08:30:40.050Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.815937, "accuracy_std_dev": 0.021042110493009022, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04137441245791246, "avg_duration_per_task_ms": 82874.20875420875, "avg_output_tokens_per_task": 27465.196969696968, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8156565, "accuracy_std_dev": 0.006011157355285233, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.028145230429292924, "avg_duration_per_task_ms": 190452.8042929293, "avg_output_tokens_per_task": 23529.740530303032, "last_run_timestamp": "2026-10-17T16:06:46.963Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8155935315487571, "accuracy_std_dev": 0.02579145123843122, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.2145485825366476, "avg_duration_per_task_ms": 977552.3288718929, "avg_output_tokens_per_task": 57980.598470363286, "last_run_timestamp": "2026-10-17T16:03:22.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.815488, "accuracy_std_dev": 0.010410666606898895, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.01878690909090909, "avg_duration_per_task_ms": 548800.3747474748, "avg_output_tokens_per_task": 18588.853535353537, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.815286480254777, "accuracy_std_dev": 0.006665997576319691, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.08723442165605096, "avg_duration_per_task_ms": 185900.4152866242, "avg_output_tokens_per_task": 24915.47388535032, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.814815, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006498378787878787, "avg_duration_per_task_ms": 589595.8333333334, "avg_output_tokens_per_task": 15681.459595959595, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.814815, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.007393566137566137, "avg_duration_per_task_ms": 549965.8571428572, "avg_output_tokens_per_task": 17741.814814814814, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.8147209999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.024774192893401017, "avg_duration_per_task_ms": 272533.6497461929, "avg_output_tokens_per_task": 20619.17766497462, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8140431250000001, "accuracy_std_dev": 0.010669274703217085, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.36383448947811453, "avg_duration_per_task_ms": 358426.1837121212, "avg_output_tokens_per_task": 22914.688552188552, "last_run_timestamp": "2026-10-17T08:22:56.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.814024, "accuracy_std_dev": null, "run_count": 1, "total_questions": 164, "avg_cost_per_task_usd": 0.010100225609756097, "avg_duration_per_task_ms": 515065.23780487804, "avg_output_tokens_per_task": 33245.84146341463, "last_run_timestamp": "2026-09-28T09:46:11.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8132656083650188, "accuracy_std_dev": 0.029437471710177493, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.08244444803548796, "avg_duration_per_task_ms": 931196.6774397972, "avg_output_tokens_per_task": 40999.92395437262, "last_run_timestamp": "2026-10-17T16:03:45.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8131313333333333, "accuracy_std_dev": 0.02212166805545088, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2550713585858586, "avg_duration_per_task_ms": 227479.37878787878, "avg_output_tokens_per_task": 25491.23232323232, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.813051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.00766168253968254, "avg_duration_per_task_ms": 356938.3703703704, "avg_output_tokens_per_task": 33904.37566137566, "last_run_timestamp": "2026-10-14T18:00:03.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8128908571428571, "accuracy_std_dev": 0.014743239462097008, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007136471861471861, "avg_duration_per_task_ms": 527522.6536796537, "avg_output_tokens_per_task": 18109.714285714286, "last_run_timestamp": "2026-10-17T16:03:10.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8124998749999999, "accuracy_std_dev": 0.02094152012413078, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04008979356060606, "avg_duration_per_task_ms": 181327.2178030303, "avg_output_tokens_per_task": 22170.065025252527, "last_run_timestamp": "2026-10-17T16:04:35.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8124578000000001, "accuracy_std_dev": 0.006784287240381237, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00604990505050505, "avg_duration_per_task_ms": 545364.5707070708, "avg_output_tokens_per_task": 18416.73232323232, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8119721658218682, "accuracy_std_dev": 0.11936196199179458, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.12433009341057204, "avg_duration_per_task_ms": 711621.3982621289, "avg_output_tokens_per_task": 38654.58653149891, "last_run_timestamp": "2026-10-17T16:04:59.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8118685, "accuracy_std_dev": 0.0618254994258033, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.042041156565656566, "avg_duration_per_task_ms": 1127766.4457070706, "avg_output_tokens_per_task": 24710.378787878788, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8115756996197718, "accuracy_std_dev": 0.04867340225790031, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.2293247072243346, "avg_duration_per_task_ms": 1102307.0380228136, "avg_output_tokens_per_task": 52484.05069708492, "last_run_timestamp": "2026-10-17T16:06:39.412Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.8112779399727148, "accuracy_std_dev": 0.010023193016698797, "run_count": 4, "total_questions": 733, "avg_cost_per_task_usd": 0.04357236971350614, "avg_duration_per_task_ms": 179585.35061391542, "avg_output_tokens_per_task": 17263.93315143247, "last_run_timestamp": "2026-10-17T08:18:01.976Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8111110666666667, "accuracy_std_dev": 0.017159518883957357, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.34046912828282827, "avg_duration_per_task_ms": 354667.00067340065, "avg_output_tokens_per_task": 22526.755218855218, "last_run_timestamp": "2026-10-17T08:02:36.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8110267499999999, "accuracy_std_dev": 0.02579051758762317, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008924909090909091, "avg_duration_per_task_ms": 574972.908459596, "avg_output_tokens_per_task": 18934.403409090908, "last_run_timestamp": "2026-10-17T16:03:42.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8103253333333333, "accuracy_std_dev": 0.011527714209195522, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04107354882154882, "avg_duration_per_task_ms": 75169.15151515152, "avg_output_tokens_per_task": 27258.996632996634, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8098576959847037, "accuracy_std_dev": 0.02469341221579907, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.25613483683875077, "avg_duration_per_task_ms": 954692.9738687061, "avg_output_tokens_per_task": 58301.671128107075, "last_run_timestamp": "2026-10-17T16:05:55.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8091166153846153, "accuracy_std_dev": 0.03183613126622014, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.19830280854700855, "avg_duration_per_task_ms": 1060541.5008547008, "avg_output_tokens_per_task": 55592.3264957265, "last_run_timestamp": "2026-10-03T16:58:49.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.8089225000000001, "accuracy_std_dev": 0.05752497129290896, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003809875420875421, "avg_duration_per_task_ms": 924304.0791245791, "avg_output_tokens_per_task": 22694.977272727272, "last_run_timestamp": "2026-10-17T16:06:12.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8084547777777777, "accuracy_std_dev": 0.02465025412885807, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11369982884399552, "avg_duration_per_task_ms": 767797.1189674523, "avg_output_tokens_per_task": 59144.13187429854, "last_run_timestamp": "2026-10-17T16:08:52.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.808081, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016894853535353537, "avg_duration_per_task_ms": 156500.61111111112, "avg_output_tokens_per_task": 6566.772727272727, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.808081, "accuracy_std_dev": 0.018027342695657245, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.044340436026936034, "avg_duration_per_task_ms": 217888.1734006734, "avg_output_tokens_per_task": 22075.030303030304, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.80780025, "accuracy_std_dev": 0.020912351996387992, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.34150990530303027, "avg_duration_per_task_ms": 356238.11531986535, "avg_output_tokens_per_task": 22606.00042087542, "last_run_timestamp": "2026-10-17T08:20:12.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.8075196666666667, "accuracy_std_dev": 0.03714760001519464, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025326144781144783, "avg_duration_per_task_ms": 166431.35521885523, "avg_output_tokens_per_task": 13572.52356902357, "last_run_timestamp": "2026-10-17T08:21:10.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8073469346289753, "accuracy_std_dev": 0.10831830089636926, "run_count": 12, "total_questions": 2264, "avg_cost_per_task_usd": 0.04317355300353357, "avg_duration_per_task_ms": 164819.90106007067, "avg_output_tokens_per_task": 20660.478798586573, "last_run_timestamp": "2026-10-17T08:17:08.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.806818, "accuracy_std_dev": 0.005877369607230755, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01103122095959596, "avg_duration_per_task_ms": 646747.8522727273, "avg_output_tokens_per_task": 11881.43686868687, "last_run_timestamp": "2026-10-17T08:23:15.092Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.806210222222222, "accuracy_std_dev": 0.03697273211969953, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.25474633557800225, "avg_duration_per_task_ms": 231596.29349046017, "avg_output_tokens_per_task": 25466.63243546577, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.8058380621468926, "accuracy_std_dev": 0.060531258531109376, "run_count": 5, "total_questions": 885, "avg_cost_per_task_usd": 0.012335033898305085, "avg_duration_per_task_ms": 403352.9514124294, "avg_output_tokens_per_task": 16083.899435028248, "last_run_timestamp": "2026-10-17T16:05:00.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.8057394503311259, "accuracy_std_dev": 0.027315871855937524, "run_count": 4, "total_questions": 604, "avg_cost_per_task_usd": 0.04681624668874173, "avg_duration_per_task_ms": 685385.5794701987, "avg_output_tokens_per_task": 20286.42715231788, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8050504, "accuracy_std_dev": 0.021411457634640384, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.3368577297979798, "avg_duration_per_task_ms": 353947.41616161616, "avg_output_tokens_per_task": 22328.277777777777, "last_run_timestamp": "2026-10-17T08:20:36.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8050312243186584, "accuracy_std_dev": 0.024402124224901022, "run_count": 4, "total_questions": 477, "avg_cost_per_task_usd": 0.0531789392033543, "avg_duration_per_task_ms": 296093.35429769394, "avg_output_tokens_per_task": 21712.16142557652, "last_run_timestamp": "2026-10-17T16:03:24.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.8045648025477706, "accuracy_std_dev": 0.09929505313576492, "run_count": 8, "total_questions": 1570, "avg_cost_per_task_usd": 0.05945578917197451, "avg_duration_per_task_ms": 1750115.5261146496, "avg_output_tokens_per_task": 19217.150955414014, "last_run_timestamp": "2026-10-17T16:03:32.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8035647794928336, "accuracy_std_dev": 0.02820532797579602, "run_count": 6, "total_questions": 907, "avg_cost_per_task_usd": 0.06920113671444322, "avg_duration_per_task_ms": 1464272.5214994487, "avg_output_tokens_per_task": 39612.85556780596, "last_run_timestamp": "2026-10-17T16:03:32.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.8033669999999999, "accuracy_std_dev": 0.005573328377190791, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.048433724242424245, "avg_duration_per_task_ms": 54676.08484848485, "avg_output_tokens_per_task": 15399.329292929293, "last_run_timestamp": "2026-10-17T08:18:07.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.802714625, "accuracy_std_dev": 0.02322035202111231, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.028997014520202022, "avg_duration_per_task_ms": 309649.94507575757, "avg_output_tokens_per_task": 19185.933712121212, "last_run_timestamp": "2026-10-17T16:09:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8020951111111111, "accuracy_std_dev": 0.027492606765554845, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04329651571268237, "avg_duration_per_task_ms": 84651.98765432098, "avg_output_tokens_per_task": 26957.542087542086, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.801499818181818, "accuracy_std_dev": 0.1018190627747603, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09262380762167126, "avg_duration_per_task_ms": 515813.015610652, "avg_output_tokens_per_task": 34167.608815427, "last_run_timestamp": "2026-10-17T16:03:43.963Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.8007853333333332, "accuracy_std_dev": 0.008398773614178595, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0439676531986532, "avg_duration_per_task_ms": 201662.1632996633, "avg_output_tokens_per_task": 21888.12962962963, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7996631666666666, "accuracy_std_dev": 0.024569953550469384, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.045980551346801346, "avg_duration_per_task_ms": 194150.27525252526, "avg_output_tokens_per_task": 21808.128787878788, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7995168260869566, "accuracy_std_dev": 0.015725089308490422, "run_count": 5, "total_questions": 966, "avg_cost_per_task_usd": 0.1727357877846791, "avg_duration_per_task_ms": 863549.4824016563, "avg_output_tokens_per_task": 55062.07867494824, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7991169867549668, "accuracy_std_dev": 0.04305133271491851, "run_count": 7, "total_questions": 1359, "avg_cost_per_task_usd": 0.14345824724061812, "avg_duration_per_task_ms": 1569797.9367181752, "avg_output_tokens_per_task": 47091.807947019865, "last_run_timestamp": "2026-10-17T16:05:55.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7989417142857143, "accuracy_std_dev": 0.13167886968217607, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15234512626262628, "avg_duration_per_task_ms": 2001837.8780663782, "avg_output_tokens_per_task": 54356.18470418471, "last_run_timestamp": "2026-10-17T16:08:20.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.798541, "accuracy_std_dev": 0.005382643216859184, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013865464646464647, "avg_duration_per_task_ms": 310914.7727272727, "avg_output_tokens_per_task": 14252.202861952863, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7979798333333333, "accuracy_std_dev": 0.04819829830819293, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1916180345117845, "avg_duration_per_task_ms": 1503115.5151515151, "avg_output_tokens_per_task": 46476.21548821549, "last_run_timestamp": "2026-10-03T17:00:45.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7975339581632652, "accuracy_std_dev": 0.14314198310659945, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.01657647346938776, "avg_duration_per_task_ms": 547902.756122449, "avg_output_tokens_per_task": 26479.933673469386, "last_run_timestamp": "2026-10-17T08:10:43.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7969696000000001, "accuracy_std_dev": 0.010464901654578508, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.013516212121212122, "avg_duration_per_task_ms": 270529.88181818184, "avg_output_tokens_per_task": 14086.99090909091, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7969274999999999, "accuracy_std_dev": 0.041893006701596396, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.026948493055555557, "avg_duration_per_task_ms": 347172.4311868687, "avg_output_tokens_per_task": 25834.242424242424, "last_run_timestamp": "2026-10-17T16:03:42.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.796581, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.06800715384615384, "avg_duration_per_task_ms": 384890.6564102564, "avg_output_tokens_per_task": 29421.871794871793, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7964659214659687, "accuracy_std_dev": 0.005460143336030668, "run_count": 8, "total_questions": 1528, "avg_cost_per_task_usd": 0.050215497382198944, "avg_duration_per_task_ms": 1089140.4594240838, "avg_output_tokens_per_task": 22865.37238219895, "last_run_timestamp": "2026-10-17T16:06:10.469Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7956228, "accuracy_std_dev": 0.021522407276138952, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006844225252525253, "avg_duration_per_task_ms": 363476.91515151516, "avg_output_tokens_per_task": 16652.98888888889, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.79541113957935, "accuracy_std_dev": 0.024018187027945608, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.2794700968769917, "avg_duration_per_task_ms": 930887.5340981517, "avg_output_tokens_per_task": 70418.68706182281, "last_run_timestamp": "2026-10-17T16:03:21.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.79523250996483, "accuracy_std_dev": 0.10177729700753506, "run_count": 5, "total_questions": 853, "avg_cost_per_task_usd": 0.014341097303634235, "avg_duration_per_task_ms": 277183.26846424385, "avg_output_tokens_per_task": 30270.443141852287, "last_run_timestamp": "2026-10-17T08:23:52.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.7950335, "accuracy_std_dev": 0.010057183614213304, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.053936401515151514, "avg_duration_per_task_ms": 418610.5353535354, "avg_output_tokens_per_task": 23400.56818181818, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.7950015744680852, "accuracy_std_dev": 0.020697025299303277, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.028578175278622088, "avg_duration_per_task_ms": 466139.9655521783, "avg_output_tokens_per_task": 23874.156028368794, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7948342770467838, "accuracy_std_dev": 0.09161678915984332, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.09734662865497076, "avg_duration_per_task_ms": 830552.8106725146, "avg_output_tokens_per_task": 29487.959795321636, "last_run_timestamp": "2026-10-17T16:06:10.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7942762, "accuracy_std_dev": 0.008274731256059032, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014654871717171718, "avg_duration_per_task_ms": 281996.1565656566, "avg_output_tokens_per_task": 14013.025252525253, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7937710000000001, "accuracy_std_dev": 0.014359120716464491, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01779377714646465, "avg_duration_per_task_ms": 59921.61174242424, "avg_output_tokens_per_task": 5776.905303030303, "last_run_timestamp": "2026-10-17T16:06:09.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7936945454545455, "accuracy_std_dev": 0.03387125348987259, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.11027911937557391, "avg_duration_per_task_ms": 415904.7438016529, "avg_output_tokens_per_task": 30768.579889807163, "last_run_timestamp": "2026-10-17T16:06:51.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.79250825, "accuracy_std_dev": 0.011190804686028344, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.004598987794612795, "avg_duration_per_task_ms": 248268.39141414143, "avg_output_tokens_per_task": 30111.32239057239, "last_run_timestamp": "2026-10-17T16:03:34.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7920900579268293, "accuracy_std_dev": 0.011512971285033247, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.01824498475609756, "avg_duration_per_task_ms": 181543.2469512195, "avg_output_tokens_per_task": 15051.83943089431, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7919672857142857, "accuracy_std_dev": 0.008482616505271861, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.19117722655122654, "avg_duration_per_task_ms": 824934.1883116884, "avg_output_tokens_per_task": 47794.95887445888, "last_run_timestamp": "2026-10-17T16:03:24.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7919196568825911, "accuracy_std_dev": 0.020933482740337288, "run_count": 10, "total_questions": 1976, "avg_cost_per_task_usd": 0.06286805870445344, "avg_duration_per_task_ms": 174709.42408906884, "avg_output_tokens_per_task": 19709.32439271255, "last_run_timestamp": "2026-10-17T08:25:47.232Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7911756164047152, "accuracy_std_dev": 0.01560584292357742, "run_count": 11, "total_questions": 2036, "avg_cost_per_task_usd": 0.0046643629666011785, "avg_duration_per_task_ms": 1348073.47151277, "avg_output_tokens_per_task": 30640.777013752457, "last_run_timestamp": "2026-10-17T16:03:22.656Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7902960850403475, "accuracy_std_dev": 0.020316089842022662, "run_count": 9, "total_questions": 1611, "avg_cost_per_task_usd": 0.040685735567970206, "avg_duration_per_task_ms": 737944.181874612, "avg_output_tokens_per_task": 18801.04593420236, "last_run_timestamp": "2026-10-17T16:06:15.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7896387272727272, "accuracy_std_dev": 0.03757319952038085, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.04347873507805325, "avg_duration_per_task_ms": 494226.878328742, "avg_output_tokens_per_task": 42653.03902662994, "last_run_timestamp": "2026-10-17T16:06:11.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7893355839160839, "accuracy_std_dev": 0.10239971274711347, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.07681546794871794, "avg_duration_per_task_ms": 733179.0168997669, "avg_output_tokens_per_task": 27895.818764568765, "last_run_timestamp": "2026-10-10T16:29:40.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7886248285229202, "accuracy_std_dev": 0.005658817544328477, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.011348117147707979, "avg_duration_per_task_ms": 785846.1052631579, "avg_output_tokens_per_task": 38718.01867572156, "last_run_timestamp": "2026-10-17T16:06:33.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7885339041353384, "accuracy_std_dev": 0.023230108247703007, "run_count": 5, "total_questions": 532, "avg_cost_per_task_usd": 0.034030992481203004, "avg_duration_per_task_ms": 116832.8590225564, "avg_output_tokens_per_task": 35347.6954887218, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.787996238141336, "accuracy_std_dev": 0.01502120070857429, "run_count": 11, "total_questions": 2066, "avg_cost_per_task_usd": 0.00856900435624395, "avg_duration_per_task_ms": 373289.4123910939, "avg_output_tokens_per_task": 33861.243949661184, "last_run_timestamp": "2026-10-17T16:07:01.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7854672791234141, "accuracy_std_dev": 0.12208478471357521, "run_count": 5, "total_questions": 867, "avg_cost_per_task_usd": 0.023352209919261825, "avg_duration_per_task_ms": 590708.8500576701, "avg_output_tokens_per_task": 19635.643598615916, "last_run_timestamp": "2026-10-17T08:18:07.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.785340410122164, "accuracy_std_dev": 0.005585783044678892, "run_count": 3, "total_questions": 573, "avg_cost_per_task_usd": 0.037465542757417106, "avg_duration_per_task_ms": 218978.30017452006, "avg_output_tokens_per_task": 37410.67888307155, "last_run_timestamp": "2026-10-17T16:06:05.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7851099187817259, "accuracy_std_dev": 0.024182763553029427, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.05363871517202482, "avg_duration_per_task_ms": 470093.5600676819, "avg_output_tokens_per_task": 23310.749576988157, "last_run_timestamp": "2026-10-17T16:03:34.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7845117999999999, "accuracy_std_dev": 0.012186465958595225, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02083039292929293, "avg_duration_per_task_ms": 462638.8303030303, "avg_output_tokens_per_task": 27689.152525252524, "last_run_timestamp": "2026-10-17T16:06:16.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7843049551569508, "accuracy_std_dev": 0.09329604363444359, "run_count": 10, "total_questions": 1115, "avg_cost_per_task_usd": 0.005419434080717489, "avg_duration_per_task_ms": 254231.5470852018, "avg_output_tokens_per_task": 21881.403587443947, "last_run_timestamp": "2026-10-17T16:03:19.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.7839820659898478, "accuracy_std_dev": 0.002844524682028023, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.013566983079526225, "avg_duration_per_task_ms": 498549.5752961083, "avg_output_tokens_per_task": 14135.162436548224, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7828282222222223, "accuracy_std_dev": 0.011110583660814166, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02110236139169473, "avg_duration_per_task_ms": 267338.53815937147, "avg_output_tokens_per_task": 12550.465207631874, "last_run_timestamp": "2026-10-17T16:03:19.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7827002835443038, "accuracy_std_dev": 0.025807959273061464, "run_count": 5, "total_questions": 790, "avg_cost_per_task_usd": 0.010936145569620253, "avg_duration_per_task_ms": 1215812.0873417722, "avg_output_tokens_per_task": 39346.51392405063, "last_run_timestamp": "2026-10-17T08:02:36.403Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.7812311687388988, "accuracy_std_dev": 0.010295797880688991, "run_count": 3, "total_questions": 563, "avg_cost_per_task_usd": 0.001959150976909414, "avg_duration_per_task_ms": 798438.0159857904, "avg_output_tokens_per_task": 12884.238010657193, "last_run_timestamp": "2026-10-17T16:05:11.871Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7807238749999998, "accuracy_std_dev": 0.039711866691322574, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1739194539141414, "avg_duration_per_task_ms": 546996.5315656565, "avg_output_tokens_per_task": 54147.905303030304, "last_run_timestamp": "2026-10-17T16:03:24.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7805356401045069, "accuracy_std_dev": 0.13188409621496255, "run_count": 8, "total_questions": 1531, "avg_cost_per_task_usd": 0.023446755062050944, "avg_duration_per_task_ms": 365068.11038536904, "avg_output_tokens_per_task": 23915.368386675374, "last_run_timestamp": "2026-10-17T16:05:01.165Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7805031924528302, "accuracy_std_dev": 0.020942354059550135, "run_count": 3, "total_questions": 265, "avg_cost_per_task_usd": 0.09049491698113207, "avg_duration_per_task_ms": 131957.70943396227, "avg_output_tokens_per_task": 37606.645283018865, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.780302875, "accuracy_std_dev": 0.044334001808537145, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.012861900883838383, "avg_duration_per_task_ms": 776646.4172979798, "avg_output_tokens_per_task": 38634.71654040404, "last_run_timestamp": "2026-10-17T08:18:06.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7799202727272726, "accuracy_std_dev": 0.017481939064962863, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0056759981634527095, "avg_duration_per_task_ms": 639660.3314967861, "avg_output_tokens_per_task": 35506.24150596878, "last_run_timestamp": "2026-10-17T16:03:16.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7796947507042253, "accuracy_std_dev": 0.11796703504024969, "run_count": 8, "total_questions": 1420, "avg_cost_per_task_usd": 0.03984405704225352, "avg_duration_per_task_ms": 145067.09014084507, "avg_output_tokens_per_task": 30331.589436619717, "last_run_timestamp": "2026-10-17T16:04:40.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7786195454545455, "accuracy_std_dev": 0.04555204390657037, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.21966554178145087, "avg_duration_per_task_ms": 867448.9150596878, "avg_output_tokens_per_task": 73058.02662993572, "last_run_timestamp": "2026-10-17T16:06:46.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7782009841269842, "accuracy_std_dev": 0.02997378382749665, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.011125198095238096, "avg_duration_per_task_ms": 566928, "avg_output_tokens_per_task": 36841.713015873014, "last_run_timestamp": "2026-10-17T16:03:22.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7774583256704981, "accuracy_std_dev": 0.04658687436660674, "run_count": 4, "total_questions": 522, "avg_cost_per_task_usd": 0.1387438639846743, "avg_duration_per_task_ms": 1171411.5785440614, "avg_output_tokens_per_task": 40017.36590038314, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7772166666666667, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013080006734006736, "avg_duration_per_task_ms": 67697.39225589225, "avg_output_tokens_per_task": 10341.513468013469, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7771783756345177, "accuracy_std_dev": 0.059728767079685954, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.007288401015228427, "avg_duration_per_task_ms": 899378.7354060913, "avg_output_tokens_per_task": 18512.973350253807, "last_run_timestamp": "2026-10-17T16:05:43.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7766553333333335, "accuracy_std_dev": 0.10513747605555944, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04385584792368125, "avg_duration_per_task_ms": 1356759.8063973065, "avg_output_tokens_per_task": 25066.917508417508, "last_run_timestamp": "2026-10-17T16:07:07.180Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.776515, "accuracy_std_dev": 0.0045135097762162586, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005179621212121213, "avg_duration_per_task_ms": 982954.0037878788, "avg_output_tokens_per_task": 28613.36111111111, "last_run_timestamp": "2026-10-17T16:08:37.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7762077565217391, "accuracy_std_dev": 0.01894682591908638, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.022503373188405798, "avg_duration_per_task_ms": 924139.8405797102, "avg_output_tokens_per_task": 38821.189855072465, "last_run_timestamp": "2026-10-17T16:08:08.338Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.775813521906226, "accuracy_std_dev": 0.014886571160397556, "run_count": 9, "total_questions": 1301, "avg_cost_per_task_usd": 0.047757321291314374, "avg_duration_per_task_ms": 944137.5941583398, "avg_output_tokens_per_task": 24378.20368946964, "last_run_timestamp": "2026-10-17T16:03:42.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.77567325, "accuracy_std_dev": 0.010934638822910419, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013401992424242422, "avg_duration_per_task_ms": 67621.48106060606, "avg_output_tokens_per_task": 10598.125, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7752524999999999, "accuracy_std_dev": 0.021901537394667075, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05750532828282828, "avg_duration_per_task_ms": 632332.7689393939, "avg_output_tokens_per_task": 23028.97095959596, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7749893918918921, "accuracy_std_dev": 0.055734405977703745, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.15352950514800515, "avg_duration_per_task_ms": 1000120.4671814672, "avg_output_tokens_per_task": 63630.38416988417, "last_run_timestamp": "2026-10-17T16:03:14.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7741301666666666, "accuracy_std_dev": 0.007235617536802828, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.019828150673400675, "avg_duration_per_task_ms": 721854.4562289562, "avg_output_tokens_per_task": 49359.81902356903, "last_run_timestamp": "2026-10-10T16:25:17.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7741139703903096, "accuracy_std_dev": 0.09718478875928833, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.004475604306864065, "avg_duration_per_task_ms": 312870.95827725437, "avg_output_tokens_per_task": 22926.07267833109, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7718855, "accuracy_std_dev": 0.03473679068005565, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05189419633838384, "avg_duration_per_task_ms": 871715.6748737374, "avg_output_tokens_per_task": 23212.847222222223, "last_run_timestamp": "2026-10-17T16:03:24.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7713499090909091, "accuracy_std_dev": 0.010348443040156889, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.014670483011937557, "avg_duration_per_task_ms": 135015.37970615242, "avg_output_tokens_per_task": 29281.215794306703, "last_run_timestamp": "2026-10-17T16:03:42.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7708977708978327, "accuracy_std_dev": 0.0364570846062103, "run_count": 8, "total_questions": 969, "avg_cost_per_task_usd": 0.012726153766769865, "avg_duration_per_task_ms": 52839.89783281734, "avg_output_tokens_per_task": 16873.008255933953, "last_run_timestamp": "2026-10-17T16:06:07.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7707982259154932, "accuracy_std_dev": 0.0072861311144724605, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.011357562816901407, "avg_duration_per_task_ms": 995701.5797183098, "avg_output_tokens_per_task": 11816.376338028169, "last_run_timestamp": "2026-10-17T16:06:12.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.770202, "accuracy_std_dev": 0.008625232557251217, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013833271043771043, "avg_duration_per_task_ms": 119359.86616161616, "avg_output_tokens_per_task": 10422.487373737375, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7691922196969697, "accuracy_std_dev": 0.017375228086595713, "run_count": 7, "total_questions": 1320, "avg_cost_per_task_usd": 0.0214991303030303, "avg_duration_per_task_ms": 238124.97348484848, "avg_output_tokens_per_task": 28579.5696969697, "last_run_timestamp": "2026-10-17T16:05:11.736Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7676768, "accuracy_std_dev": 0.018594973916625974, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015932259595959595, "avg_duration_per_task_ms": 441129.9383838384, "avg_output_tokens_per_task": 16024.338383838383, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7676531051401869, "accuracy_std_dev": 0.011937876359567938, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.01795761292834891, "avg_duration_per_task_ms": 102616.90381619938, "avg_output_tokens_per_task": 5949.5965732087225, "last_run_timestamp": "2026-10-17T08:25:18.584Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7671156666666666, "accuracy_std_dev": 0.03093018545405481, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013063415824915825, "avg_duration_per_task_ms": 936092.1515151515, "avg_output_tokens_per_task": 46359.02693602694, "last_run_timestamp": "2026-10-10T08:15:34.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7664968651509212, "accuracy_std_dev": 0.016788365025716064, "run_count": 13, "total_questions": 2551, "avg_cost_per_task_usd": 0.007439355154841239, "avg_duration_per_task_ms": 466577.11054488434, "avg_output_tokens_per_task": 27687.860446883577, "last_run_timestamp": "2026-10-17T16:03:42.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7664264643091028, "accuracy_std_dev": 0.008541751796323752, "run_count": 8, "total_questions": 1527, "avg_cost_per_task_usd": 0.024611972495088407, "avg_duration_per_task_ms": 71012.95219384413, "avg_output_tokens_per_task": 25773.081859855927, "last_run_timestamp": "2026-10-17T16:06:09.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7660061679586564, "accuracy_std_dev": 0.07448120643864166, "run_count": 6, "total_questions": 1161, "avg_cost_per_task_usd": 0.18327362360034452, "avg_duration_per_task_ms": 1579588.0456503015, "avg_output_tokens_per_task": 43785.853574504734, "last_run_timestamp": "2026-10-17T16:07:01.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7659344612403101, "accuracy_std_dev": 0.01074604027072297, "run_count": 4, "total_questions": 774, "avg_cost_per_task_usd": 0.004193177002583979, "avg_duration_per_task_ms": 4103.266149870801, "avg_output_tokens_per_task": 2323.702842377261, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.7659312647058824, "accuracy_std_dev": 0.018460106716917977, "run_count": 4, "total_questions": 544, "avg_cost_per_task_usd": 0.008997586397058823, "avg_duration_per_task_ms": 678227.3382352941, "avg_output_tokens_per_task": 29866.545955882353, "last_run_timestamp": "2026-10-17T16:03:22.651Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7646065075987841, "accuracy_std_dev": 0.016409967770839785, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.056628046605876385, "avg_duration_per_task_ms": 583978.3080040527, "avg_output_tokens_per_task": 26120.428571428572, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.76431, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017299227272727275, "avg_duration_per_task_ms": 73383.17171717172, "avg_output_tokens_per_task": 5944.843434343435, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7631440720576461, "accuracy_std_dev": 0.033247584998763444, "run_count": 8, "total_questions": 1249, "avg_cost_per_task_usd": 0.06387943394715773, "avg_duration_per_task_ms": 833616.5316253003, "avg_output_tokens_per_task": 53539.40272217774, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.7630475000000001, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004469671717171717, "avg_duration_per_task_ms": 567471.6388888889, "avg_output_tokens_per_task": 29671.171717171717, "last_run_timestamp": "2026-10-17T16:08:23.187Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7622055000000001, "accuracy_std_dev": 0.008018844134287667, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.020353424242424242, "avg_duration_per_task_ms": 1224475.143939394, "avg_output_tokens_per_task": 61537.31691919192, "last_run_timestamp": "2026-10-17T16:06:04.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.7620653333333334, "accuracy_std_dev": 0.00883726150393256, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007579895622895623, "avg_duration_per_task_ms": 139557.0117845118, "avg_output_tokens_per_task": 30124.809764309764, "last_run_timestamp": "2026-10-17T16:06:03.146Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.011624017325155306, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.020688753246753247, "avg_duration_per_task_ms": 1247623.8001443, "avg_output_tokens_per_task": 51654.42640692641, "last_run_timestamp": "2026-10-17T16:03:24.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7606508265895953, "accuracy_std_dev": 0.044831136424469266, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.27017780154142584, "avg_duration_per_task_ms": 889465.8985228003, "avg_output_tokens_per_task": 74206.37700706488, "last_run_timestamp": "2026-10-17T16:03:34.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7605754720812183, "accuracy_std_dev": 0.010447569992586773, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0025078527918781725, "avg_duration_per_task_ms": 100160.06345177664, "avg_output_tokens_per_task": 31388.34771573604, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.75925925, "accuracy_std_dev": 0.006627946132664344, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017339029040404044, "avg_duration_per_task_ms": 75536.6938131313, "avg_output_tokens_per_task": 28657.905303030304, "last_run_timestamp": "2026-10-17T16:06:10.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7587235606060606, "accuracy_std_dev": 0.03366533728388146, "run_count": 8, "total_questions": 1452, "avg_cost_per_task_usd": 0.007390471074380165, "avg_duration_per_task_ms": 981052.2692837466, "avg_output_tokens_per_task": 19052.973829201102, "last_run_timestamp": "2026-10-17T16:06:14.585Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.758698, "accuracy_std_dev": 0.012009662443216293, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.056970961279461274, "avg_duration_per_task_ms": 99388.35016835017, "avg_output_tokens_per_task": 22699.07239057239, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.758698, "accuracy_std_dev": 0.0082855039677741, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015277286195286194, "avg_duration_per_task_ms": 1138394.335016835, "avg_output_tokens_per_task": 54274.19528619529, "last_run_timestamp": "2026-10-17T08:20:47.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7586279375000001, "accuracy_std_dev": 0.013868153578742699, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.018340981376262628, "avg_duration_per_task_ms": 158098.73516414143, "avg_output_tokens_per_task": 30346.9375, "last_run_timestamp": "2026-10-17T16:04:59.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7584345607287448, "accuracy_std_dev": 0.07943384181065398, "run_count": 5, "total_questions": 988, "avg_cost_per_task_usd": 0.01775063663967611, "avg_duration_per_task_ms": 1537506.3319838056, "avg_output_tokens_per_task": 17162.693319838058, "last_run_timestamp": "2026-10-17T16:14:35.207Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7578456145038168, "accuracy_std_dev": 0.01544961858701939, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.01670373155216285, "avg_duration_per_task_ms": 254619.6227735369, "avg_output_tokens_per_task": 28145.37468193384, "last_run_timestamp": "2026-10-17T16:11:05.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7576013994334279, "accuracy_std_dev": 0.11080526429777046, "run_count": 9, "total_questions": 1765, "avg_cost_per_task_usd": 0.15527972067988668, "avg_duration_per_task_ms": 887994.8634560907, "avg_output_tokens_per_task": 41127.971104815864, "last_run_timestamp": "2026-10-18T01:36:49.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7575756666666668, "accuracy_std_dev": 0.0064470965471977405, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022524166666666668, "avg_duration_per_task_ms": 96254.67171717172, "avg_output_tokens_per_task": 14855.659932659933, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7573266413662239, "accuracy_std_dev": 0.03112028552949828, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.1659841872232764, "avg_duration_per_task_ms": 1040091.2226438962, "avg_output_tokens_per_task": 75445.36812144212, "last_run_timestamp": "2026-10-17T16:03:32.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7571548999999999, "accuracy_std_dev": 0.12945420603166974, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01753179696969697, "avg_duration_per_task_ms": 657106.2575757576, "avg_output_tokens_per_task": 36803.128282828286, "last_run_timestamp": "2026-10-17T08:23:46.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.7558923750000001, "accuracy_std_dev": 0.009411207626249413, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.018470106691919192, "avg_duration_per_task_ms": 142999.88257575757, "avg_output_tokens_per_task": 30551.185606060608, "last_run_timestamp": "2026-10-17T16:07:16.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7554704302103251, "accuracy_std_dev": 0.05900138631408906, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.2827103932441045, "avg_duration_per_task_ms": 965963.304652645, "avg_output_tokens_per_task": 81541.58444869344, "last_run_timestamp": "2026-10-17T16:03:32.461Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7551755057034221, "accuracy_std_dev": 0.031175143900028734, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.004692086185044361, "avg_duration_per_task_ms": 725750.8219264892, "avg_output_tokens_per_task": 28116.21609632446, "last_run_timestamp": "2026-10-17T16:06:08.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7547698333333334, "accuracy_std_dev": 0.0766049855914889, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004288400673400673, "avg_duration_per_task_ms": 447675.2971380471, "avg_output_tokens_per_task": 15291.021885521886, "last_run_timestamp": "2026-10-17T08:26:03.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7545171538940811, "accuracy_std_dev": 0.03419234805098943, "run_count": 10, "total_questions": 1605, "avg_cost_per_task_usd": 0.04500766791277259, "avg_duration_per_task_ms": 265542.445482866, "avg_output_tokens_per_task": 16958.05109034268, "last_run_timestamp": "2026-10-17T08:22:59.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.753788, "accuracy_std_dev": 0.02000377094449942, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07918905176767678, "avg_duration_per_task_ms": 855570.0353535353, "avg_output_tokens_per_task": 22639.340909090908, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.75084367721519, "accuracy_std_dev": 0.024667232906226022, "run_count": 4, "total_questions": 790, "avg_cost_per_task_usd": 0.020310108860759495, "avg_duration_per_task_ms": 155074.11012658227, "avg_output_tokens_per_task": 11752.967088607595, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7508415000000002, "accuracy_std_dev": 0.0622895, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06288509848484848, "avg_duration_per_task_ms": 493754.3712121212, "avg_output_tokens_per_task": 26228.825757575756, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.75076231097561, "accuracy_std_dev": 0.10261490128027215, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.006507009654471546, "avg_duration_per_task_ms": 566183.218495935, "avg_output_tokens_per_task": 27810.678353658535, "last_run_timestamp": "2026-10-10T08:22:53.947Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7500000000000001, "accuracy_std_dev": 0.14215609823271738, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.029263358585858587, "avg_duration_per_task_ms": 133657.62815656565, "avg_output_tokens_per_task": 22413.67297979798, "last_run_timestamp": "2026-10-17T16:06:08.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.75, "accuracy_std_dev": 0.038766301716181284, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05368984785353535, "avg_duration_per_task_ms": 976528.220959596, "avg_output_tokens_per_task": 27872.888257575756, "last_run_timestamp": "2026-10-17T16:03:34.731Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.7493987142857143, "accuracy_std_dev": 0.00923961159842752, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.013690483405483407, "avg_duration_per_task_ms": 96167.38383838384, "avg_output_tokens_per_task": 15049.160173160173, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7485971111111113, "accuracy_std_dev": 0.012497660327210088, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.017153025252525253, "avg_duration_per_task_ms": 1346127.6537598204, "avg_output_tokens_per_task": 55391.88832772166, "last_run_timestamp": "2026-10-17T16:05:03.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7485289698857738, "accuracy_std_dev": 0.027178252463320746, "run_count": 5, "total_questions": 963, "avg_cost_per_task_usd": 0.01670032710280374, "avg_duration_per_task_ms": 258104.83489096572, "avg_output_tokens_per_task": 53131.220145379026, "last_run_timestamp": "2026-10-17T08:04:11.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7485157480916032, "accuracy_std_dev": 0.032894842349333445, "run_count": 4, "total_questions": 786, "avg_cost_per_task_usd": 0.014386558524173029, "avg_duration_per_task_ms": 1033415.2888040713, "avg_output_tokens_per_task": 51046.291348600505, "last_run_timestamp": "2026-10-17T08:02:02.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.7481013645569621, "accuracy_std_dev": 0.004003499999999993, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.010130237974683544, "avg_duration_per_task_ms": 298779.6734177215, "avg_output_tokens_per_task": 60721.07341772152, "last_run_timestamp": "2026-10-10T08:39:12.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018268570707070705, "avg_duration_per_task_ms": 31005.68686868687, "avg_output_tokens_per_task": 5873.606060606061, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7467534285714286, "accuracy_std_dev": 0.009517321679925722, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015003951659451658, "avg_duration_per_task_ms": 107319.33261183261, "avg_output_tokens_per_task": 14469.299422799422, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7467264444444446, "accuracy_std_dev": 0.05912320253901284, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.01319890179573513, "avg_duration_per_task_ms": 100190.80527497194, "avg_output_tokens_per_task": 5733.753647586981, "last_run_timestamp": "2026-10-17T08:31:20.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7466665885714285, "accuracy_std_dev": 0.012433924485430002, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.11517983936507933, "avg_duration_per_task_ms": 766437.2444444444, "avg_output_tokens_per_task": 44974.1346031746, "last_run_timestamp": "2026-10-17T16:06:03.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7465043071718538, "accuracy_std_dev": 0.027768881506803414, "run_count": 6, "total_questions": 739, "avg_cost_per_task_usd": 0.053211952638700946, "avg_duration_per_task_ms": 424838.5182679296, "avg_output_tokens_per_task": 15554.012178619756, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7462993119730186, "accuracy_std_dev": 0.0800254867207247, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.00489628442945475, "avg_duration_per_task_ms": 596052.9263631253, "avg_output_tokens_per_task": 32164.360314783586, "last_run_timestamp": "2026-10-17T16:03:21.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7457915, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015561896464646466, "avg_duration_per_task_ms": 220958.36363636365, "avg_output_tokens_per_task": 61761.72474747475, "last_run_timestamp": "2026-10-17T16:04:31.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.744318125, "accuracy_std_dev": 0.008684208274758005, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006803909090909091, "avg_duration_per_task_ms": 163196.51830808082, "avg_output_tokens_per_task": 7906.570075757576, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7435209222520109, "accuracy_std_dev": 0.004720778410977948, "run_count": 6, "total_questions": 1119, "avg_cost_per_task_usd": 0.008935632707774798, "avg_duration_per_task_ms": 139189.39052725647, "avg_output_tokens_per_task": 11357.61126005362, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.743266, "accuracy_std_dev": 0.006355106254029135, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005297381944444444, "avg_duration_per_task_ms": 155629.39141414143, "avg_output_tokens_per_task": 8558.156565656565, "last_run_timestamp": "2026-10-17T16:06:15.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7404603333333333, "accuracy_std_dev": 0.009642303833743399, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.023280140572390574, "avg_duration_per_task_ms": 99017.15824915825, "avg_output_tokens_per_task": 15411.767676767677, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7393379999999999, "accuracy_std_dev": 0.01649975226076642, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.016316385521885524, "avg_duration_per_task_ms": 860370.430976431, "avg_output_tokens_per_task": 57893.19191919192, "last_run_timestamp": "2026-10-17T08:03:50.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7391490737652341, "accuracy_std_dev": 0.06249035866035656, "run_count": 8, "total_questions": 1559, "avg_cost_per_task_usd": 0.2797374688903143, "avg_duration_per_task_ms": 1317138.1584348942, "avg_output_tokens_per_task": 88993.17318794099, "last_run_timestamp": "2026-10-17T16:03:22.651Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.7388167142857143, "accuracy_std_dev": 0.011146245808910405, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00482988961038961, "avg_duration_per_task_ms": 107324.14213564213, "avg_output_tokens_per_task": 7874.049062049062, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7385620777988616, "accuracy_std_dev": 0.1239043242359175, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.005195311827956989, "avg_duration_per_task_ms": 366943.59392789373, "avg_output_tokens_per_task": 15666.573055028462, "last_run_timestamp": "2026-10-17T16:06:04.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.738309, "accuracy_std_dev": 0.008136215650479735, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.059655349046015715, "avg_duration_per_task_ms": 125601.6632996633, "avg_output_tokens_per_task": 23853.966891133558, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7372685, "accuracy_std_dev": 0.06687866148481143, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.11904529797979799, "avg_duration_per_task_ms": 603565.316919192, "avg_output_tokens_per_task": 35752.68939393939, "last_run_timestamp": "2026-10-17T16:06:16.211Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7361669504685407, "accuracy_std_dev": 0.0072054719473726965, "run_count": 8, "total_questions": 1494, "avg_cost_per_task_usd": 0.006662820615796521, "avg_duration_per_task_ms": 235800.40361445784, "avg_output_tokens_per_task": 8241.185408299865, "last_run_timestamp": "2026-10-17T16:03:45.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7349895701863354, "accuracy_std_dev": 0.017162011317797845, "run_count": 6, "total_questions": 805, "avg_cost_per_task_usd": 0.015730058385093167, "avg_duration_per_task_ms": 68792.66708074535, "avg_output_tokens_per_task": 9043.119254658384, "last_run_timestamp": "2026-10-17T16:03:44.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7343086374045802, "accuracy_std_dev": 0.014863350526021203, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.006568683842239185, "avg_duration_per_task_ms": 349096.14885496185, "avg_output_tokens_per_task": 26082.53117048346, "last_run_timestamp": "2026-10-17T16:06:42.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7340773735119047, "accuracy_std_dev": 0.017792537262244897, "run_count": 17, "total_questions": 3360, "avg_cost_per_task_usd": 0.006654232738095238, "avg_duration_per_task_ms": 121514.80714285714, "avg_output_tokens_per_task": 6940.3625, "last_run_timestamp": "2026-10-17T16:03:44.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.734007, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0222420303030303, "avg_duration_per_task_ms": 980476.9090909091, "avg_output_tokens_per_task": 55433.85858585859, "last_run_timestamp": "2026-10-17T08:02:36.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.733048471794872, "accuracy_std_dev": 0.05454575358179184, "run_count": 6, "total_questions": 1170, "avg_cost_per_task_usd": 0.017418347008547008, "avg_duration_per_task_ms": 1056799.6769230769, "avg_output_tokens_per_task": 44136.11538461538, "last_run_timestamp": "2026-10-17T16:06:05.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7312169142857143, "accuracy_std_dev": 0.012725630006007392, "run_count": 35, "total_questions": 6930, "avg_cost_per_task_usd": 0.2044193023088022, "avg_duration_per_task_ms": 243801.0481962482, "avg_output_tokens_per_task": 38178.343001443005, "last_run_timestamp": "2026-10-17T08:18:07.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7310605833333333, "accuracy_std_dev": 0.015416108968966707, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1858813265993266, "avg_duration_per_task_ms": 297454.41077441076, "avg_output_tokens_per_task": 37007.64057239057, "last_run_timestamp": "2026-10-17T08:19:37.981Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.7308918662420382, "accuracy_std_dev": 0.007761999999999991, "run_count": 2, "total_questions": 314, "avg_cost_per_task_usd": 0.007396050955414012, "avg_duration_per_task_ms": 197815.93949044586, "avg_output_tokens_per_task": 9168.713375796178, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7297980000000001, "accuracy_std_dev": 0.01108520326509943, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006000632756132756, "avg_duration_per_task_ms": 399051.6572871573, "avg_output_tokens_per_task": 7941.9047619047615, "last_run_timestamp": "2026-10-17T16:07:29.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7285024173913045, "accuracy_std_dev": 0.053992787395869, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.2980392717391304, "avg_duration_per_task_ms": 1335107.6862318842, "avg_output_tokens_per_task": 94830.67536231884, "last_run_timestamp": "2026-10-17T16:09:49.789Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7282548333333334, "accuracy_std_dev": 0.009283646839410062, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19013492424242423, "avg_duration_per_task_ms": 242933.63425925927, "avg_output_tokens_per_task": 37836.16708754209, "last_run_timestamp": "2026-10-17T08:18:06.163Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7281143333333333, "accuracy_std_dev": 0.016544932312409815, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004527595117845118, "avg_duration_per_task_ms": 163776.3367003367, "avg_output_tokens_per_task": 7680.468013468014, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7271808509532062, "accuracy_std_dev": 0.03540222199908109, "run_count": 6, "total_questions": 1154, "avg_cost_per_task_usd": 0.011152869150779895, "avg_duration_per_task_ms": 504501.5831889082, "avg_output_tokens_per_task": 55506.62998266898, "last_run_timestamp": "2026-10-10T08:24:27.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7271424333619948, "accuracy_std_dev": 0.012738961511398356, "run_count": 12, "total_questions": 2326, "avg_cost_per_task_usd": 0.1868387523645744, "avg_duration_per_task_ms": 345994.4789337919, "avg_output_tokens_per_task": 37177.46560619088, "last_run_timestamp": "2026-10-17T08:18:07.188Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.7269411348511384, "accuracy_std_dev": 0.05322474791235121, "run_count": 6, "total_questions": 1142, "avg_cost_per_task_usd": 0.0670280718038529, "avg_duration_per_task_ms": 1772246.9842381787, "avg_output_tokens_per_task": 34290.80998248686, "last_run_timestamp": "2026-10-17T16:09:10.586Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.726852, "accuracy_std_dev": 0.015898785165854648, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005457933712121213, "avg_duration_per_task_ms": 255088.5498737374, "avg_output_tokens_per_task": 7917.640782828283, "last_run_timestamp": "2026-10-17T16:03:34.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7261503333333333, "accuracy_std_dev": 0.004418546090690373, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.059926168350168355, "avg_duration_per_task_ms": 123042.70033670033, "avg_output_tokens_per_task": 23940.483164983165, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7250370931445603, "accuracy_std_dev": 0.10506379222881687, "run_count": 8, "total_questions": 1342, "avg_cost_per_task_usd": 0.0044079307004470936, "avg_duration_per_task_ms": 545749.8681073026, "avg_output_tokens_per_task": 25961.05439642325, "last_run_timestamp": "2026-10-17T08:02:36.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.723064, "accuracy_std_dev": 0.02312882263540449, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0057709015151515145, "avg_duration_per_task_ms": 342190.50694444444, "avg_output_tokens_per_task": 17394.30808080808, "last_run_timestamp": "2026-10-17T16:03:21.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7215208055555558, "accuracy_std_dev": 0.013006390565622906, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.197375834315376, "avg_duration_per_task_ms": 267372.5809483726, "avg_output_tokens_per_task": 36812.16091470258, "last_run_timestamp": "2026-10-17T08:26:18.338Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7209128888888889, "accuracy_std_dev": 0.010210347185787597, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00438738608305275, "avg_duration_per_task_ms": 649291.1077441077, "avg_output_tokens_per_task": 7853.185746352413, "last_run_timestamp": "2026-10-17T16:03:10.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.7173201760037348, "accuracy_std_dev": 0.0373953514862056, "run_count": 11, "total_questions": 2142, "avg_cost_per_task_usd": 0.010640089169000933, "avg_duration_per_task_ms": 264502.14985994395, "avg_output_tokens_per_task": 47791.74136321195, "last_run_timestamp": "2026-10-17T08:02:36.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.7163299999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005710287878787878, "avg_duration_per_task_ms": 197152.55555555556, "avg_output_tokens_per_task": 6940.146464646465, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7160493333333333, "accuracy_std_dev": 0.011690806112307026, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.018215316498316498, "avg_duration_per_task_ms": 375673.00505050505, "avg_output_tokens_per_task": 64702.65151515151, "last_run_timestamp": "2026-10-17T08:18:00.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7154881249999999, "accuracy_std_dev": 0.05526776053550001, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006628352272727273, "avg_duration_per_task_ms": 109434.6811868687, "avg_output_tokens_per_task": 16300.07133838384, "last_run_timestamp": "2026-10-17T16:03:44.040Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7135241666666668, "accuracy_std_dev": 0.005271695534223852, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015626640572390573, "avg_duration_per_task_ms": 96955.32070707071, "avg_output_tokens_per_task": 10336.517676767677, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7135240000000002, "accuracy_std_dev": 0.009836496361340617, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007830127104377103, "avg_duration_per_task_ms": 79042.44696969698, "avg_output_tokens_per_task": 9942.877946127946, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7121211666666666, "accuracy_std_dev": 0.009066017803068557, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011008058080808082, "avg_duration_per_task_ms": 95816.33922558922, "avg_output_tokens_per_task": 10395.001683501683, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.71170025, "accuracy_std_dev": 0.018456257615982147, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0649322058080808, "avg_duration_per_task_ms": 1066951.6603535353, "avg_output_tokens_per_task": 21770.531565656565, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7112403023255814, "accuracy_std_dev": 0.09623599999999999, "run_count": 2, "total_questions": 344, "avg_cost_per_task_usd": 0.06576875, "avg_duration_per_task_ms": 744084.4970930233, "avg_output_tokens_per_task": 27095.636627906977, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.7107745999999999, "accuracy_std_dev": 0.05091182812510271, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006678358585858586, "avg_duration_per_task_ms": 125158.29696969697, "avg_output_tokens_per_task": 7409.80808080808, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.709650171648164, "accuracy_std_dev": 0.01649939376664086, "run_count": 6, "total_questions": 1171, "avg_cost_per_task_usd": 0.010711521776259607, "avg_duration_per_task_ms": 103546.38514090521, "avg_output_tokens_per_task": 9904.411614005125, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7093155, "accuracy_std_dev": 0.008890530969332882, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008779646464646466, "avg_duration_per_task_ms": 132997.85521885523, "avg_output_tokens_per_task": 5690.617845117845, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7091377451737451, "accuracy_std_dev": 0.09862411167716228, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.13626344465894466, "avg_duration_per_task_ms": 996229.6081081082, "avg_output_tokens_per_task": 72903.59588159589, "last_run_timestamp": "2026-10-17T16:03:22.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.707752053820598, "accuracy_std_dev": 0.25911579823070297, "run_count": 8, "total_questions": 1505, "avg_cost_per_task_usd": 0.025081889036544854, "avg_duration_per_task_ms": 430774.81063122925, "avg_output_tokens_per_task": 14989.863122923589, "last_run_timestamp": "2026-10-17T16:03:44.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7064695714285715, "accuracy_std_dev": 0.05204993264948887, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.016059919913419915, "avg_duration_per_task_ms": 997721.5901875902, "avg_output_tokens_per_task": 48256.89682539683, "last_run_timestamp": "2026-10-17T16:03:19.083Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7059358649303454, "accuracy_std_dev": 0.04834769030284051, "run_count": 17, "total_questions": 3302, "avg_cost_per_task_usd": 0.008049274076317382, "avg_duration_per_task_ms": 519119.62113870383, "avg_output_tokens_per_task": 25312.240763173835, "last_run_timestamp": "2026-10-17T16:11:22.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.705263, "accuracy_std_dev": null, "run_count": 1, "total_questions": 95, "avg_cost_per_task_usd": 0.10938622105263157, "avg_duration_per_task_ms": 1384847.8, "avg_output_tokens_per_task": 40599.36842105263, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7045453333333335, "accuracy_std_dev": 0.009510928024587757, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00597795707070707, "avg_duration_per_task_ms": 110020.64814814815, "avg_output_tokens_per_task": 5546.97138047138, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.7037037142857143, "accuracy_std_dev": 0.04201543115405606, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007533322510822511, "avg_duration_per_task_ms": 804813.1904761905, "avg_output_tokens_per_task": 25640.617604617604, "last_run_timestamp": "2026-10-17T16:06:47.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7031426666666668, "accuracy_std_dev": 0.014373068248947025, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011147152356902357, "avg_duration_per_task_ms": 198021.7255892256, "avg_output_tokens_per_task": 5803.353535353535, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7027678642599278, "accuracy_std_dev": 0.009533289144800297, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.010358495306859207, "avg_duration_per_task_ms": 145686.42599277978, "avg_output_tokens_per_task": 9350.431768953069, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.7011786666666667, "accuracy_std_dev": 0.0268613535999626, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.010966026094276094, "avg_duration_per_task_ms": 64637.51430976431, "avg_output_tokens_per_task": 9084.513468013469, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.7008102604166667, "accuracy_std_dev": 0.02755789234312128, "run_count": 9, "total_questions": 1728, "avg_cost_per_task_usd": 0.009351708912037038, "avg_duration_per_task_ms": 375939.1927083333, "avg_output_tokens_per_task": 45073.12384259259, "last_run_timestamp": "2026-10-17T08:02:36.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.700785612121212, "accuracy_std_dev": 0.15090828820097987, "run_count": 8, "total_questions": 1485, "avg_cost_per_task_usd": 0.08803936228956229, "avg_duration_per_task_ms": 618855.5521885522, "avg_output_tokens_per_task": 27400.163636363635, "last_run_timestamp": "2026-10-17T16:05:47.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7000472153628653, "accuracy_std_dev": 0.1118059188262249, "run_count": 12, "total_questions": 2122, "avg_cost_per_task_usd": 0.03202059802073515, "avg_duration_per_task_ms": 1048480.0966069746, "avg_output_tokens_per_task": 33620.651743638075, "last_run_timestamp": "2026-10-17T16:06:17.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6995251282660333, "accuracy_std_dev": 0.037727051631422225, "run_count": 5, "total_questions": 842, "avg_cost_per_task_usd": 0.03918485154394299, "avg_duration_per_task_ms": 785269.9073634205, "avg_output_tokens_per_task": 41981.14845605701, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.695286, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007596338383838384, "avg_duration_per_task_ms": 2114924.02020202, "avg_output_tokens_per_task": 54058.00505050505, "last_run_timestamp": "2026-10-10T16:26:33.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6938431428571429, "accuracy_std_dev": 0.009541538965844799, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005521870129870129, "avg_duration_per_task_ms": 387910.27056277054, "avg_output_tokens_per_task": 6090.331168831169, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.6927610000000001, "accuracy_std_dev": 0.01993714402817014, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.009704229797979799, "avg_duration_per_task_ms": 259526.76094276094, "avg_output_tokens_per_task": 6456.316498316498, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6898100478128179, "accuracy_std_dev": 0.04728305630022238, "run_count": 10, "total_questions": 1966, "avg_cost_per_task_usd": 0.007569284333672431, "avg_duration_per_task_ms": 285452.8184130214, "avg_output_tokens_per_task": 40484.10834181078, "last_run_timestamp": "2026-10-17T08:02:36.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6894712328966521, "accuracy_std_dev": 0.029494685027795925, "run_count": 6, "total_questions": 687, "avg_cost_per_task_usd": 0.009409835516739446, "avg_duration_per_task_ms": 56014.804949053854, "avg_output_tokens_per_task": 6093.893740902475, "last_run_timestamp": "2026-10-17T16:03:30.488Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.6848486000000001, "accuracy_std_dev": 0.06736876735580072, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0077578979797979794, "avg_duration_per_task_ms": 1345818.3151515152, "avg_output_tokens_per_task": 61748.48484848485, "last_run_timestamp": "2026-10-17T16:03:44.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6814815, "accuracy_std_dev": 0.13220309825813464, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.6080289767676768, "avg_duration_per_task_ms": 627829.4717171717, "avg_output_tokens_per_task": 40460.77222222222, "last_run_timestamp": "2026-10-17T08:21:26.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6790182687747036, "accuracy_std_dev": 0.01730772007660164, "run_count": 8, "total_questions": 1012, "avg_cost_per_task_usd": 0.004263678853754942, "avg_duration_per_task_ms": 849188.9940711462, "avg_output_tokens_per_task": 42493.400197628456, "last_run_timestamp": "2026-10-17T16:05:46.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.6777017260273973, "accuracy_std_dev": 0.01773685922201921, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.0021049600456621003, "avg_duration_per_task_ms": 197982.70319634702, "avg_output_tokens_per_task": 4056.279109589041, "last_run_timestamp": "2026-10-17T16:06:14.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.6748038333333333, "accuracy_std_dev": 0.04521916160735648, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01522573063973064, "avg_duration_per_task_ms": 286760.11026936024, "avg_output_tokens_per_task": 8293.19696969697, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6747943299556681, "accuracy_std_dev": 0.09586293689486765, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.011517654844838505, "avg_duration_per_task_ms": 960521.5490816972, "avg_output_tokens_per_task": 33713.77960734642, "last_run_timestamp": "2026-10-17T16:03:32.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6735395649484536, "accuracy_std_dev": 0.02076995482325371, "run_count": 5, "total_questions": 485, "avg_cost_per_task_usd": 0.003951113402061856, "avg_duration_per_task_ms": 479082.6536082474, "avg_output_tokens_per_task": 26380.544329896908, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6714038275698719, "accuracy_std_dev": 0.14105760450673585, "run_count": 32, "total_questions": 6333, "avg_cost_per_task_usd": 0.6613663402810673, "avg_duration_per_task_ms": 596949.4503394916, "avg_output_tokens_per_task": 41226.86957208274, "last_run_timestamp": "2026-10-17T08:25:40.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6708499999999998, "accuracy_std_dev": 0.13629141592324748, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.6572281983471076, "avg_duration_per_task_ms": 465247.270278543, "avg_output_tokens_per_task": 40899.25971839608, "last_run_timestamp": "2026-10-17T08:20:14.789Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.6698808181818181, "accuracy_std_dev": 0.13895384854941661, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.6170192970615244, "avg_duration_per_task_ms": 632348.5018365473, "avg_output_tokens_per_task": 40978.85583103765, "last_run_timestamp": "2026-10-17T08:03:37.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.6672790000000001, "accuracy_std_dev": 0.1453197674941212, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.6195322176308541, "avg_duration_per_task_ms": 633753.1198347107, "avg_output_tokens_per_task": 41213.71395775941, "last_run_timestamp": "2026-10-17T08:21:05.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6662457499999999, "accuracy_std_dev": 0.017551141520923935, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005718719696969698, "avg_duration_per_task_ms": 566951.7815656565, "avg_output_tokens_per_task": 16218.064393939394, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6661054999999999, "accuracy_std_dev": 0.006849761936252494, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002126188552188552, "avg_duration_per_task_ms": 243791.89646464647, "avg_output_tokens_per_task": 3027.095117845118, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6656563999999999, "accuracy_std_dev": 0.00579280583482652, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0027868222222222224, "avg_duration_per_task_ms": 87457.03737373737, "avg_output_tokens_per_task": 2951.019191919192, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6652237142857144, "accuracy_std_dev": 0.016133353975221514, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005764501443001443, "avg_duration_per_task_ms": 425162.0974025974, "avg_output_tokens_per_task": 15800.678932178933, "last_run_timestamp": "2026-10-17T16:05:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6651514999999998, "accuracy_std_dev": 0.008019320822239275, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.005088861616161617, "avg_duration_per_task_ms": 104484.55757575757, "avg_output_tokens_per_task": 9583.160606060606, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.664469578782172, "accuracy_std_dev": 0.017175728905408912, "run_count": 9, "total_questions": 1593, "avg_cost_per_task_usd": 0.006976930947897049, "avg_duration_per_task_ms": 485628.49654739484, "avg_output_tokens_per_task": 49654.3157564344, "last_run_timestamp": "2026-10-17T16:12:05.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6643207216890595, "accuracy_std_dev": 0.02405113813802531, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.027497225847728728, "avg_duration_per_task_ms": 839162.8976327576, "avg_output_tokens_per_task": 18575.21817018554, "last_run_timestamp": "2026-10-17T16:06:25.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6637718184210527, "accuracy_std_dev": 0.20207857374716895, "run_count": 10, "total_questions": 1900, "avg_cost_per_task_usd": 0.012164477368421052, "avg_duration_per_task_ms": 311613.41736842104, "avg_output_tokens_per_task": 14454.243684210527, "last_run_timestamp": "2026-10-17T08:17:57.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.66372075, "accuracy_std_dev": 0.00844903779654822, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00248682702020202, "avg_duration_per_task_ms": 87766.55681818182, "avg_output_tokens_per_task": 2304.1906565656564, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6607744999999999, "accuracy_std_dev": 0.011572094721786573, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015565636363636365, "avg_duration_per_task_ms": 14159.460858585859, "avg_output_tokens_per_task": 1751.3977272727273, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6582491666666667, "accuracy_std_dev": 0.024492817507837335, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007911382154882154, "avg_duration_per_task_ms": 819880.6978114478, "avg_output_tokens_per_task": 54362.767676767675, "last_run_timestamp": "2026-10-17T16:06:46.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6576179999999999, "accuracy_std_dev": 0.01601415369290554, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.008870517992424245, "avg_duration_per_task_ms": 321713.3065025252, "avg_output_tokens_per_task": 58815.57323232323, "last_run_timestamp": "2026-10-17T16:03:45.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.6560053838120105, "accuracy_std_dev": 0.20012163968993332, "run_count": 4, "total_questions": 766, "avg_cost_per_task_usd": 0.07259215535248041, "avg_duration_per_task_ms": 1457323.1005221931, "avg_output_tokens_per_task": 31100.218015665796, "last_run_timestamp": "2026-10-17T16:05:04.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6554433333333334, "accuracy_std_dev": 0.0216322289497253, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007501081369248036, "avg_duration_per_task_ms": 643870.0549943884, "avg_output_tokens_per_task": 58023.832772166104, "last_run_timestamp": "2026-10-17T16:06:15.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6552547006369427, "accuracy_std_dev": 0.08012703749983525, "run_count": 4, "total_questions": 628, "avg_cost_per_task_usd": 0.06530196178343949, "avg_duration_per_task_ms": 1671084.4410828026, "avg_output_tokens_per_task": 24249.23566878981, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.6550925, "accuracy_std_dev": 0.13950088584306553, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.014581559974747476, "avg_duration_per_task_ms": 426923.34027777775, "avg_output_tokens_per_task": 13349.987373737375, "last_run_timestamp": "2026-10-17T16:03:19.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.651601, "accuracy_std_dev": null, "run_count": 1, "total_questions": 177, "avg_cost_per_task_usd": 0.03645816384180791, "avg_duration_per_task_ms": 1143037.5875706214, "avg_output_tokens_per_task": 29354.60451977401, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.651515, "accuracy_std_dev": 0.1905456270048375, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.058297676767676776, "avg_duration_per_task_ms": 1098090.5791245792, "avg_output_tokens_per_task": 23174.20202020202, "last_run_timestamp": "2026-10-17T16:03:44.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6505050000000001, "accuracy_std_dev": 0.008356827340564098, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003142527272727273, "avg_duration_per_task_ms": 40777.49797979798, "avg_output_tokens_per_task": 2977.0545454545454, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.650363823580786, "accuracy_std_dev": 0.007103203700131033, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.0036315545851528385, "avg_duration_per_task_ms": 58503.03493449782, "avg_output_tokens_per_task": 2202.661135371179, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6502977410714286, "accuracy_std_dev": 0.027563000000000004, "run_count": 2, "total_questions": 224, "avg_cost_per_task_usd": 0.00407284375, "avg_duration_per_task_ms": 238632.88839285713, "avg_output_tokens_per_task": 13352.90625, "last_run_timestamp": "2026-10-09T03:20:25.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.011195005920300148, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004075954545454546, "avg_duration_per_task_ms": 28668.510101010103, "avg_output_tokens_per_task": 2354.649831649832, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6478116, "accuracy_std_dev": 0.01301437818107341, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0023702696969696967, "avg_duration_per_task_ms": 65368.43434343435, "avg_output_tokens_per_task": 2772.5838383838386, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6473065000000001, "accuracy_std_dev": 0.009707729665065862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00413114393939394, "avg_duration_per_task_ms": 27377.72601010101, "avg_output_tokens_per_task": 2388.9305555555557, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6470586811145511, "accuracy_std_dev": 0.06899576470516142, "run_count": 4, "total_questions": 646, "avg_cost_per_task_usd": 0.006973526315789473, "avg_duration_per_task_ms": 477220.6222910217, "avg_output_tokens_per_task": 36135.41950464396, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6464646666666667, "accuracy_std_dev": 0.013537798302858865, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015368191919191919, "avg_duration_per_task_ms": 14060.520202020201, "avg_output_tokens_per_task": 1727.3535353535353, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6459467692307692, "accuracy_std_dev": 0.14204828227220914, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.6158103986013985, "avg_duration_per_task_ms": 628254.9867909868, "avg_output_tokens_per_task": 40928.243589743586, "last_run_timestamp": "2026-10-17T08:10:38.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.6454544, "accuracy_std_dev": 0.002519632084253531, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.002477851515151515, "avg_duration_per_task_ms": 59827.33737373738, "avg_output_tokens_per_task": 2673.8050505050505, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.643659, "accuracy_std_dev": 0.015752669318774726, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015761657407407408, "avg_duration_per_task_ms": 18961.11531986532, "avg_output_tokens_per_task": 1683.8872053872053, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6428171666666667, "accuracy_std_dev": 0.012039053318494584, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002693207070707071, "avg_duration_per_task_ms": 142395.0521885522, "avg_output_tokens_per_task": 9633.525252525253, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6420455, "accuracy_std_dev": 0.02154191080196928, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007707547348484848, "avg_duration_per_task_ms": 627203.6717171717, "avg_output_tokens_per_task": 45830.878787878784, "last_run_timestamp": "2026-10-17T16:03:19.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.6419754999999999, "accuracy_std_dev": 0.013802963036367705, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004449612794612795, "avg_duration_per_task_ms": 74990.68265993266, "avg_output_tokens_per_task": 8364.945286195287, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6419754444444444, "accuracy_std_dev": 0.014018014458475362, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007092590347923681, "avg_duration_per_task_ms": 241593.06509539843, "avg_output_tokens_per_task": 11377.103254769921, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6417703596127247, "accuracy_std_dev": 0.22170503732098648, "run_count": 8, "total_questions": 1446, "avg_cost_per_task_usd": 0.008234107883817428, "avg_duration_per_task_ms": 190026.56362378976, "avg_output_tokens_per_task": 13566.77316735823, "last_run_timestamp": "2026-10-17T16:05:13.856Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6405723333333334, "accuracy_std_dev": 0.0030349496792460334, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004210166666666667, "avg_duration_per_task_ms": 30431.28367003367, "avg_output_tokens_per_task": 2308.9638047138046, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6404886237845923, "accuracy_std_dev": 0.019812264339163707, "run_count": 7, "total_questions": 1337, "avg_cost_per_task_usd": 0.005841909498878086, "avg_duration_per_task_ms": 853617.8878085265, "avg_output_tokens_per_task": 35274.41361256545, "last_run_timestamp": "2026-10-17T08:18:07.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.638266870496592, "accuracy_std_dev": 0.013425056404805994, "run_count": 6, "total_questions": 1027, "avg_cost_per_task_usd": 0.007463308666017527, "avg_duration_per_task_ms": 34605.769230769234, "avg_output_tokens_per_task": 12362.268743914314, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6374194522580645, "accuracy_std_dev": 0.03485686397164123, "run_count": 8, "total_questions": 1550, "avg_cost_per_task_usd": 0.059443716774193536, "avg_duration_per_task_ms": 1133345.7683870967, "avg_output_tokens_per_task": 25152.790967741934, "last_run_timestamp": "2026-10-17T16:05:03.429Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6368495215189873, "accuracy_std_dev": 0.014462691735980549, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0026009772151898737, "avg_duration_per_task_ms": 238051.80337552744, "avg_output_tokens_per_task": 8445.996624472575, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6338480836550836, "accuracy_std_dev": 0.014864426946152353, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.016590933719433723, "avg_duration_per_task_ms": 90048.76833976834, "avg_output_tokens_per_task": 55073.25611325612, "last_run_timestamp": "2026-10-17T16:06:05.694Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6336718477157359, "accuracy_std_dev": 0.012529818338002093, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.003363994923857868, "avg_duration_per_task_ms": 229990.32571912013, "avg_output_tokens_per_task": 10944.545685279189, "last_run_timestamp": "2026-10-03T16:41:22.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6334780000000001, "accuracy_std_dev": 0.013155152146592608, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0016263744588744587, "avg_duration_per_task_ms": 75470.49134199134, "avg_output_tokens_per_task": 8221.989177489177, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6325426993295513, "accuracy_std_dev": 0.010534965676735725, "run_count": 10, "total_questions": 1939, "avg_cost_per_task_usd": 0.01766013512119649, "avg_duration_per_task_ms": 478451.96183599794, "avg_output_tokens_per_task": 88282.73388344508, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6311448, "accuracy_std_dev": 0.007685307265165154, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0011036777777777777, "avg_duration_per_task_ms": 412979.17272727273, "avg_output_tokens_per_task": 11552.624242424243, "last_run_timestamp": "2026-10-17T16:07:02.132Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6299665, "accuracy_std_dev": 0.033651413370763474, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0068110621212121225, "avg_duration_per_task_ms": 109012.8005050505, "avg_output_tokens_per_task": 9497.993434343434, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6285795064570466, "accuracy_std_dev": 0.020919173358428876, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01718929421673217, "avg_duration_per_task_ms": 631165.4817518248, "avg_output_tokens_per_task": 83912.1010668164, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6272744175576814, "accuracy_std_dev": 0.010599488803187111, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.0018089758019133374, "avg_duration_per_task_ms": 225302.9572312887, "avg_output_tokens_per_task": 9913.687675858188, "last_run_timestamp": "2026-10-17T16:06:43.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.6253005714285714, "accuracy_std_dev": 0.007302792065996677, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.024268393217893217, "avg_duration_per_task_ms": 1032337.5678210679, "avg_output_tokens_per_task": 100917.8658008658, "last_run_timestamp": "2026-10-03T17:27:37.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.6200336919191919, "accuracy_std_dev": 0.11339640181763927, "run_count": 8, "total_questions": 990, "avg_cost_per_task_usd": 0.09412290202020202, "avg_duration_per_task_ms": 1259060.601010101, "avg_output_tokens_per_task": 26810.946464646466, "last_run_timestamp": "2026-10-17T16:09:29.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.6196141015067407, "accuracy_std_dev": 0.08983333259674553, "run_count": 8, "total_questions": 1261, "avg_cost_per_task_usd": 0.0522221165741475, "avg_duration_per_task_ms": 323929.8643933386, "avg_output_tokens_per_task": 19801.716098334655, "last_run_timestamp": "2026-10-17T16:08:25.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6182949425287356, "accuracy_std_dev": 0.0013475000000000015, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.011105244252873564, "avg_duration_per_task_ms": 345775.4454022989, "avg_output_tokens_per_task": 38606.68103448276, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6170940030769231, "accuracy_std_dev": 0.014320214565431623, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.015717948717948717, "avg_duration_per_task_ms": 780521.6543589743, "avg_output_tokens_per_task": 16712.84717948718, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6128760749682338, "accuracy_std_dev": 0.007225322047320813, "run_count": 4, "total_questions": 787, "avg_cost_per_task_usd": 0.0033133367217280814, "avg_duration_per_task_ms": 404332.3011435832, "avg_output_tokens_per_task": 13398.312579415502, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6099888333333333, "accuracy_std_dev": 0.01106802965627677, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003627024410774411, "avg_duration_per_task_ms": 33220.15151515151, "avg_output_tokens_per_task": 2140.8367003367002, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6099860008438818, "accuracy_std_dev": 0.004922906311880752, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0040446168776371305, "avg_duration_per_task_ms": 27156.491983122363, "avg_output_tokens_per_task": 2148.8616033755275, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007801348484848485, "avg_duration_per_task_ms": 208074.1717171717, "avg_output_tokens_per_task": 15081.5, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.608586, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007727368686868687, "avg_duration_per_task_ms": 263769.8611111111, "avg_output_tokens_per_task": 15259.444444444445, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6062142522796352, "accuracy_std_dev": 0.009864001918085761, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.008205017223910841, "avg_duration_per_task_ms": 297243.1023302938, "avg_output_tokens_per_task": 17225.689969604864, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Mara", "accuracy": 0.6061323018867925, "accuracy_std_dev": 0.029270000000000018, "run_count": 2, "total_questions": 212, "avg_cost_per_task_usd": 0.034763094339622644, "avg_duration_per_task_ms": 71660.68867924529, "avg_output_tokens_per_task": 7461.462264150943, "last_run_timestamp": "2026-10-17T16:06:00.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6059645129698252, "accuracy_std_dev": 0.012733602453351516, "run_count": 10, "total_questions": 1889, "avg_cost_per_task_usd": 0.008509499205929062, "avg_duration_per_task_ms": 554697.4531498147, "avg_output_tokens_per_task": 13980.221281101112, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6050506, "accuracy_std_dev": 0.015451550117706637, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005770865656565657, "avg_duration_per_task_ms": 636718.6202020202, "avg_output_tokens_per_task": 19268.987878787877, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.604377, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00791619191919192, "avg_duration_per_task_ms": 146373.4797979798, "avg_output_tokens_per_task": 15059.80303030303, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6042256518448439, "accuracy_std_dev": 0.02010368015853481, "run_count": 6, "total_questions": 1057, "avg_cost_per_task_usd": 0.0021907710501419112, "avg_duration_per_task_ms": 51585.36707663198, "avg_output_tokens_per_task": 2058.464522232734, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6038161666666666, "accuracy_std_dev": 0.007251665174672309, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0035006599326599325, "avg_duration_per_task_ms": 56534.7601010101, "avg_output_tokens_per_task": 2119.1574074074074, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6035354999999999, "accuracy_std_dev": 0.0052117790868377945, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.010990611952861952, "avg_duration_per_task_ms": 32954.444444444445, "avg_output_tokens_per_task": 2093.7045454545455, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.602339, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0316934269005848, "avg_duration_per_task_ms": 1131594.5263157894, "avg_output_tokens_per_task": 37899.070175438595, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6021274927659573, "accuracy_std_dev": 0.0818342643735224, "run_count": 6, "total_questions": 1175, "avg_cost_per_task_usd": 0.012776367659574468, "avg_duration_per_task_ms": 1356678.9268085107, "avg_output_tokens_per_task": 56278.53021276596, "last_run_timestamp": "2026-10-17T16:06:06.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.6016879113924051, "accuracy_std_dev": 0.05594315696758628, "run_count": 4, "total_questions": 790, "avg_cost_per_task_usd": 0.033461643037974685, "avg_duration_per_task_ms": 2683325.098734177, "avg_output_tokens_per_task": 25317.36075949367, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.6003243382761816, "accuracy_std_dev": 0.19105188493471462, "run_count": 11, "total_questions": 2158, "avg_cost_per_task_usd": 0.02540118164967563, "avg_duration_per_task_ms": 704415.6756255792, "avg_output_tokens_per_task": 35888.73818350324, "last_run_timestamp": "2026-10-17T16:04:35.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.5994523122629583, "accuracy_std_dev": 0.10371686124425177, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.009700670670037928, "avg_duration_per_task_ms": 326884.85082174465, "avg_output_tokens_per_task": 27706.066371681416, "last_run_timestamp": "2026-10-17T16:05:58.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.599327, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014991068181818182, "avg_duration_per_task_ms": 284433.01515151514, "avg_output_tokens_per_task": 12302.075757575758, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5973626666666666, "accuracy_std_dev": 0.014397638598819675, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003072445286195286, "avg_duration_per_task_ms": 655562.0867003367, "avg_output_tokens_per_task": 13595.005050505051, "last_run_timestamp": "2026-10-07T13:51:26.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.5962405000000001, "accuracy_std_dev": 0.005702057428390789, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003268709595959596, "avg_duration_per_task_ms": 65099.85437710438, "avg_output_tokens_per_task": 2080.245791245791, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5948373333333333, "accuracy_std_dev": 0.010316724200162651, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04306378282828283, "avg_duration_per_task_ms": 401108.6296296296, "avg_output_tokens_per_task": 21434.68181818182, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.5947809999999999, "accuracy_std_dev": 0.13542944894076767, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0028764909090909087, "avg_duration_per_task_ms": 219197.7287878788, "avg_output_tokens_per_task": 6791.85202020202, "last_run_timestamp": "2026-10-17T16:05:42.381Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5942281016850668, "accuracy_std_dev": 0.00965016140081236, "run_count": 9, "total_questions": 1721, "avg_cost_per_task_usd": 0.009055742010459035, "avg_duration_per_task_ms": 523063.0418361418, "avg_output_tokens_per_task": 45085.088901801275, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.591751, "accuracy_std_dev": 0.006627755628164546, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017204867003367003, "avg_duration_per_task_ms": 118615.4696969697, "avg_output_tokens_per_task": 42873.59259259259, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5906884318766067, "accuracy_std_dev": 0.010206288841526426, "run_count": 6, "total_questions": 1167, "avg_cost_per_task_usd": 0.006399965724078834, "avg_duration_per_task_ms": 683293.4078834619, "avg_output_tokens_per_task": 42407.72150814053, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5902780000000002, "accuracy_std_dev": 0.009063399582937982, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010274607954545456, "avg_duration_per_task_ms": 11527.349747474747, "avg_output_tokens_per_task": 13245.006313131313, "last_run_timestamp": "2026-10-17T16:12:37.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5841753333333334, "accuracy_std_dev": 0.033949349496304376, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02777477777777778, "avg_duration_per_task_ms": 1412449.8375420875, "avg_output_tokens_per_task": 69325.78367003366, "last_run_timestamp": "2026-10-13T21:28:41.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.5816397652439025, "accuracy_std_dev": 0.007030422274657478, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.025769568089430896, "avg_duration_per_task_ms": 1126074.3231707318, "avg_output_tokens_per_task": 67814.54166666667, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.5774414999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010289136363636365, "avg_duration_per_task_ms": 331432.74242424243, "avg_output_tokens_per_task": 40883.856060606064, "last_run_timestamp": "2026-10-17T16:06:05.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.575269, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.03680247311827957, "avg_duration_per_task_ms": 1058115.6344086023, "avg_output_tokens_per_task": 47809.01612903226, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5696305846528623, "accuracy_std_dev": 0.02980472884459779, "run_count": 8, "total_questions": 821, "avg_cost_per_task_usd": 0.03290161875761267, "avg_duration_per_task_ms": 61744.40560292327, "avg_output_tokens_per_task": 7044.583434835567, "last_run_timestamp": "2026-10-17T16:05:03.577Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5621250282776349, "accuracy_std_dev": 0.06329693423460571, "run_count": 4, "total_questions": 389, "avg_cost_per_task_usd": 0.04400786118251928, "avg_duration_per_task_ms": 513471.6503856041, "avg_output_tokens_per_task": 21046.47557840617, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5572391666666667, "accuracy_std_dev": 0.009959776661094785, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004621723905723906, "avg_duration_per_task_ms": 448955.2752525252, "avg_output_tokens_per_task": 16279.704545454546, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.5512932188317349, "accuracy_std_dev": 0.02565312085410966, "run_count": 6, "total_questions": 1147, "avg_cost_per_task_usd": 0.005740845684394072, "avg_duration_per_task_ms": 730028.2885789014, "avg_output_tokens_per_task": 14883.85701830863, "last_run_timestamp": "2026-10-13T09:46:38.737Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5505053333333333, "accuracy_std_dev": 0.016890890559378122, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.017397131313131312, "avg_duration_per_task_ms": 129348.1271043771, "avg_output_tokens_per_task": 43400.656565656565, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5488216666666668, "accuracy_std_dev": 0.006298902038362641, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0171891531986532, "avg_duration_per_task_ms": 121387.12121212122, "avg_output_tokens_per_task": 42892.78114478115, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.535985, "accuracy_std_dev": 0.011835000612589788, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0018682651515151514, "avg_duration_per_task_ms": 92043.92676767676, "avg_output_tokens_per_task": 1839.9911616161617, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.53367, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00486480303030303, "avg_duration_per_task_ms": 109732.54545454546, "avg_output_tokens_per_task": 10482.964646464647, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.5331086666666667, "accuracy_std_dev": 0.05754091454106565, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005119237373737374, "avg_duration_per_task_ms": 101325.84511784512, "avg_output_tokens_per_task": 5420.193602693603, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.532164, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.03933312280701755, "avg_duration_per_task_ms": 1068622.9473684211, "avg_output_tokens_per_task": 51714.59064327485, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5238096808510638, "accuracy_std_dev": 0.00755476843854264, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0166975390070922, "avg_duration_per_task_ms": 736031.547112462, "avg_output_tokens_per_task": 42415.456940222895, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5230117451776649, "accuracy_std_dev": 0.08115411707264149, "run_count": 5, "total_questions": 985, "avg_cost_per_task_usd": 0.0036350548223350256, "avg_duration_per_task_ms": 131062.38578680203, "avg_output_tokens_per_task": 3887.226395939086, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.522126, "accuracy_std_dev": 0.02474951208754282, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01514130735930736, "avg_duration_per_task_ms": 13491.472582972583, "avg_output_tokens_per_task": 1320.8239538239538, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.513468, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0047852878787878785, "avg_duration_per_task_ms": 118138.42929292929, "avg_output_tokens_per_task": 10671.39898989899, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.506823, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.03426515789473684, "avg_duration_per_task_ms": 1056421.3684210526, "avg_output_tokens_per_task": 40494.5730994152, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5065361568627451, "accuracy_std_dev": 0.1358455723807, "run_count": 5, "total_questions": 918, "avg_cost_per_task_usd": 0.004705715686274509, "avg_duration_per_task_ms": 112349.47167755991, "avg_output_tokens_per_task": 4227.696078431372, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.5056116666666668, "accuracy_std_dev": 0.007570922855827376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012942734006734005, "avg_duration_per_task_ms": 12612.308080808081, "avg_output_tokens_per_task": 1100.3198653198654, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5, "accuracy_std_dev": 0.015120700182200542, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008362104377104377, "avg_duration_per_task_ms": 19874.548821548822, "avg_output_tokens_per_task": 1898.5521885521885, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.49831644444444445, "accuracy_std_dev": 0.012943393871530763, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0023515011223344553, "avg_duration_per_task_ms": 10501.347362514029, "avg_output_tokens_per_task": 2329.722222222222, "last_run_timestamp": "2026-10-17T16:05:59.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.49764320000000006, "accuracy_std_dev": 0.01852187333290023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0008495191919191919, "avg_duration_per_task_ms": 13033.880808080808, "avg_output_tokens_per_task": 1851.6353535353535, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4966332500000001, "accuracy_std_dev": 0.014773079898501197, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008230315656565656, "avg_duration_per_task_ms": 17739.39898989899, "avg_output_tokens_per_task": 1863.909090909091, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.49197393750000007, "accuracy_std_dev": 0.015797164320219007, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.01112520286885246, "avg_duration_per_task_ms": 456396.28176229505, "avg_output_tokens_per_task": 27747.69262295082, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.48962583163265305, "accuracy_std_dev": 0.02041486033848873, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.011151638775510205, "avg_duration_per_task_ms": 495714.47040816327, "avg_output_tokens_per_task": 27939.333673469388, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.48550299999999996, "accuracy_std_dev": 0.005230926367926303, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.002357327721661055, "avg_duration_per_task_ms": 14866.148709315376, "avg_output_tokens_per_task": 2216.58810325477, "last_run_timestamp": "2026-10-17T16:07:34.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.48224677424483314, "accuracy_std_dev": 0.016591290595765484, "run_count": 13, "total_questions": 2516, "avg_cost_per_task_usd": 0.0011952321144674085, "avg_duration_per_task_ms": 57709.6359300477, "avg_output_tokens_per_task": 2204.5071542130368, "last_run_timestamp": "2026-10-17T16:04:59.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.48091999999999996, "accuracy_std_dev": 0.004418715424192887, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.048067654882154885, "avg_duration_per_task_ms": 46868.4898989899, "avg_output_tokens_per_task": 16842.27946127946, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.48042899999999994, "accuracy_std_dev": 0.00882577897411893, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0025970202020202024, "avg_duration_per_task_ms": 51590.09785353536, "avg_output_tokens_per_task": 2849.220328282828, "last_run_timestamp": "2026-10-17T16:03:42.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47777759999999997, "accuracy_std_dev": 0.010931021024588695, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0010385141414141412, "avg_duration_per_task_ms": 206806.0606060606, "avg_output_tokens_per_task": 2333.3515151515153, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.4758696666666667, "accuracy_std_dev": 0.012035777397225144, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.003142085858585859, "avg_duration_per_task_ms": 61848.09259259259, "avg_output_tokens_per_task": 2215.2059483726152, "last_run_timestamp": "2026-10-17T16:05:55.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4753785, "accuracy_std_dev": 0.01205013506148375, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013200202020202019, "avg_duration_per_task_ms": 46402.35037878788, "avg_output_tokens_per_task": 2362.8636363636365, "last_run_timestamp": "2026-10-17T16:06:14.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.4717298835443038, "accuracy_std_dev": 0.015344517267985329, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.0013212088607594936, "avg_duration_per_task_ms": 90902.26202531645, "avg_output_tokens_per_task": 3031.067088607595, "last_run_timestamp": "2026-10-17T16:03:46.179Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.4715905, "accuracy_std_dev": 0.008925495476442747, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013712070707070706, "avg_duration_per_task_ms": 71157.06376262626, "avg_output_tokens_per_task": 2259.597222222222, "last_run_timestamp": "2026-10-17T16:03:19.113Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.46742483824768327, "accuracy_std_dev": 0.006948085005956674, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.002137747262005055, "avg_duration_per_task_ms": 34994.02274641955, "avg_output_tokens_per_task": 2587.217354675653, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4634305555555556, "accuracy_std_dev": 0.017035579793877875, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0007985993265993267, "avg_duration_per_task_ms": 162610.1588103255, "avg_output_tokens_per_task": 2244.9444444444443, "last_run_timestamp": "2026-10-17T16:05:59.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.45862900000000006, "accuracy_std_dev": null, "run_count": 1, "total_questions": 141, "avg_cost_per_task_usd": 0.0011888297872340425, "avg_duration_per_task_ms": 278701.0070921986, "avg_output_tokens_per_task": 7719.879432624113, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4320986666666667, "accuracy_std_dev": 0.00781631474687543, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010060690235690235, "avg_duration_per_task_ms": 20918.510101010103, "avg_output_tokens_per_task": 1483.5286195286196, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4315373333333334, "accuracy_std_dev": 0.010588221658469791, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010723358585858589, "avg_duration_per_task_ms": 21765.176767676767, "avg_output_tokens_per_task": 1505.858585858586, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4305555, "accuracy_std_dev": 0.007469952225416168, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010262436868686867, "avg_duration_per_task_ms": 17602.7702020202, "avg_output_tokens_per_task": 1517.9292929292928, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.422559, "accuracy_std_dev": 0.09221941654915557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019149646464646466, "avg_duration_per_task_ms": 3630846.4646464647, "avg_output_tokens_per_task": 17674.203703703704, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.40225624573378843, "accuracy_std_dev": 0.0951137316667801, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.004076047781569966, "avg_duration_per_task_ms": 40782.70307167235, "avg_output_tokens_per_task": 1714.6023890784984, "last_run_timestamp": "2026-10-17T16:03:30.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.38720499999999997, "accuracy_std_dev": 0.010647388881786938, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00015740101010101008, "avg_duration_per_task_ms": 6591.242424242424, "avg_output_tokens_per_task": 937.0363636363636, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.37878799999999996, "accuracy_std_dev": 0.01683499999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000344469696969697, "avg_duration_per_task_ms": 46431.74494949495, "avg_output_tokens_per_task": 2036.358585858586, "last_run_timestamp": "2026-10-07T13:35:26.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.31216928571428576, "accuracy_std_dev": 0.09915241498206441, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008725867965367964, "avg_duration_per_task_ms": 624172.6608946609, "avg_output_tokens_per_task": 21583.0202020202, "last_run_timestamp": "2026-10-17T16:03:19.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2891415, "accuracy_std_dev": 0.003232916137174003, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006112626262626263, "avg_duration_per_task_ms": 10017.040404040405, "avg_output_tokens_per_task": 577.7083333333334, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.286826625, "accuracy_std_dev": 0.017534761211216273, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00147922601010101, "avg_duration_per_task_ms": 134135.83712121213, "avg_output_tokens_per_task": 20106.075757575756, "last_run_timestamp": "2026-10-17T16:06:11.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.280724, "accuracy_std_dev": 0.010959107970085884, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0028003642676767675, "avg_duration_per_task_ms": 53003.86363636364, "avg_output_tokens_per_task": 35001.6029040404, "last_run_timestamp": "2026-10-17T16:03:32.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.27476285969556585, "accuracy_std_dev": 0.016441799164020953, "run_count": 8, "total_questions": 1511, "avg_cost_per_task_usd": 0.008752005956320317, "avg_duration_per_task_ms": 357755.06022501655, "avg_output_tokens_per_task": 39075.03904698875, "last_run_timestamp": "2026-10-17T16:06:52.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2739361560283688, "accuracy_std_dev": 0.008153796102974918, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.00006610283687943263, "avg_duration_per_task_ms": 8879.929078014184, "avg_output_tokens_per_task": 551.2074468085107, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.273148, "accuracy_std_dev": 0.006113657170630353, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006268181818181819, "avg_duration_per_task_ms": 9553.181818181818, "avg_output_tokens_per_task": 586.439393939394, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-4b-it", "model_slug": "google/gemma-3-4b-it", "model_name": "Google: Gemma 3 4B", "provider_name": "Parasail", "accuracy": 0.271044, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011306565656565657, "avg_duration_per_task_ms": 10846.11616161616, "avg_output_tokens_per_task": 1804.4646464646464, "last_run_timestamp": "2026-10-15T03:00:03.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.2700681612244898, "accuracy_std_dev": 0.016752437809465226, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.00005762346938775511, "avg_duration_per_task_ms": 9734.870408163266, "avg_output_tokens_per_task": 576.8010204081633, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.26941151269035535, "accuracy_std_dev": 0.011962128387354623, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.0003863474337281444, "avg_duration_per_task_ms": 404376.6192893401, "avg_output_tokens_per_task": 9255.347433728144, "last_run_timestamp": "2026-10-17T16:03:34.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.26430966666666666, "accuracy_std_dev": 0.1964870751683739, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014085552188552189, "avg_duration_per_task_ms": 155316.38383838383, "avg_output_tokens_per_task": 16743.055555555555, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.25907233333333335, "accuracy_std_dev": 0.0140503088854935, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0004220207631874298, "avg_duration_per_task_ms": 68331.25420875421, "avg_output_tokens_per_task": 8116.787878787879, "last_run_timestamp": "2026-10-17T16:05:03.496Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.22962980000000002, "accuracy_std_dev": 0.11415863629423752, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0012938323232323233, "avg_duration_per_task_ms": 537044.9343434343, "avg_output_tokens_per_task": 4183.114141414141, "last_run_timestamp": "2026-08-29T17:16:14.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.1861954, "accuracy_std_dev": 0.12390642146491036, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.001524880808080808, "avg_duration_per_task_ms": 49007.062626262625, "avg_output_tokens_per_task": 1318.2202020202021, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.17340049999999999, "accuracy_std_dev": 0.11279449999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00031717929292929297, "avg_duration_per_task_ms": 9334.631313131313, "avg_output_tokens_per_task": 867.929292929293, "last_run_timestamp": "2026-10-13T18:56:05.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.0025255000000000004, "accuracy_std_dev": 0.0008415000000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034195530303030306, "avg_duration_per_task_ms": 159765.32575757575, "avg_output_tokens_per_task": 3219.0782828282827, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034544065656565656, "avg_duration_per_task_ms": 164367.952020202, "avg_output_tokens_per_task": 3253.931818181818, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8368423578947368, "accuracy_std_dev": 0.014073500000000017, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.008221831578947368, "avg_duration_per_task_ms": 31565.694736842106, "avg_output_tokens_per_task": 9295.157894736842, "last_run_timestamp": "2026-10-10T11:02:59.339Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.817778, "accuracy_std_dev": 0.008314764498569197, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.045532706666666665, "avg_duration_per_task_ms": 81385.46, "avg_output_tokens_per_task": 5393.9, "last_run_timestamp": "2026-10-17T11:16:25.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8115940000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05019476086956522, "avg_duration_per_task_ms": 222941.58695652173, "avg_output_tokens_per_task": 15139.565217391304, "last_run_timestamp": "2026-10-10T19:11:12.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.8083334999999999, "accuracy_std_dev": 0.011666499999999969, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.27321468, "avg_duration_per_task_ms": 27944.19, "avg_output_tokens_per_task": 12314.55, "last_run_timestamp": "2026-10-17T10:26:07.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8054545454545454, "accuracy_std_dev": 0.018161693915609792, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8776261163636364, "avg_duration_per_task_ms": 132507.47090909092, "avg_output_tokens_per_task": 5799.549090909091, "last_run_timestamp": "2026-10-17T10:21:30.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.805, "accuracy_std_dev": 0.0050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03878186, "avg_duration_per_task_ms": 112134.29, "avg_output_tokens_per_task": 4615.95, "last_run_timestamp": "2026-10-17T10:59:57.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8022223333333333, "accuracy_std_dev": 0.022222300000974995, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.87912498, "avg_duration_per_task_ms": 142260.38, "avg_output_tokens_per_task": 5752.837777777778, "last_run_timestamp": "2026-10-17T11:00:09.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8018181818181818, "accuracy_std_dev": 0.026222075532643652, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.9189436672727272, "avg_duration_per_task_ms": 125382.45454545454, "avg_output_tokens_per_task": 5783.118181818182, "last_run_timestamp": "2026-10-17T10:30:43.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.8, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01127516, "avg_duration_per_task_ms": 269406.38, "avg_output_tokens_per_task": 14001.05, "last_run_timestamp": "2026-10-17T19:00:44.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.7983335000000001, "accuracy_std_dev": 0.008660350267165889, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.05923889500000001, "avg_duration_per_task_ms": 937672.915, "avg_output_tokens_per_task": 41561.56, "last_run_timestamp": "2026-10-17T11:22:17.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7957576363636364, "accuracy_std_dev": 0.016641845981920093, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8206288509090909, "avg_duration_per_task_ms": 133285.35818181818, "avg_output_tokens_per_task": 5774.987272727272, "last_run_timestamp": "2026-10-17T10:25:10.036Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7944445714285714, "accuracy_std_dev": 0.021757085246773614, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.9448550457142856, "avg_duration_per_task_ms": 95959.28857142857, "avg_output_tokens_per_task": 5694.378095238095, "last_run_timestamp": "2026-10-17T10:59:43.833Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.7942177959183673, "accuracy_std_dev": 0.014513999999999971, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.06639602040816327, "avg_duration_per_task_ms": 463491.29591836734, "avg_output_tokens_per_task": 8929.387755102041, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7936708987341772, "accuracy_std_dev": 0.03037925132800345, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.12511070886075948, "avg_duration_per_task_ms": 2933394.1088607595, "avg_output_tokens_per_task": 27206.47088607595, "last_run_timestamp": "2026-10-17T19:41:14.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.7933334999999999, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11020131, "avg_duration_per_task_ms": 320831.42, "avg_output_tokens_per_task": 18660.61, "last_run_timestamp": "2026-10-17T10:59:44.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01272006, "avg_duration_per_task_ms": 187174.32, "avg_output_tokens_per_task": 12168.72, "last_run_timestamp": "2026-10-18T01:15:30.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01333386, "avg_duration_per_task_ms": 71979.7, "avg_output_tokens_per_task": 13242.08, "last_run_timestamp": "2026-10-18T00:09:45.214Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00828808, "avg_duration_per_task_ms": 196026.56, "avg_output_tokens_per_task": 16303, "last_run_timestamp": "2026-10-14T19:21:35.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03146312, "avg_duration_per_task_ms": 636210.72, "avg_output_tokens_per_task": 16745.44, "last_run_timestamp": "2026-10-14T19:17:59.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7900492128514055, "accuracy_std_dev": 0.014761537132245638, "run_count": 15, "total_questions": 747, "avg_cost_per_task_usd": 0.10236340696117804, "avg_duration_per_task_ms": 137138.95448460508, "avg_output_tokens_per_task": 11684.20749665328, "last_run_timestamp": "2026-10-17T11:15:32.561Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7897823718592966, "accuracy_std_dev": 0.03282945408637456, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.196944148241206, "avg_duration_per_task_ms": 804508.3291457286, "avg_output_tokens_per_task": 33320.5351758794, "last_run_timestamp": "2026-10-17T19:42:06.819Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.788889, "accuracy_std_dev": 0.019115994210782406, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015915486666666666, "avg_duration_per_task_ms": 190491.50666666665, "avg_output_tokens_per_task": 11843.433333333332, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7883334999999999, "accuracy_std_dev": 0.013642112089042542, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17747285999999998, "avg_duration_per_task_ms": 72082.685, "avg_output_tokens_per_task": 6332.95, "last_run_timestamp": "2026-10-17T10:24:14.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.788, "accuracy_std_dev": 0.021039613432760593, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.10081700199999999, "avg_duration_per_task_ms": 73090.95, "avg_output_tokens_per_task": 4784.634, "last_run_timestamp": "2026-10-17T11:21:57.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7878787272727273, "accuracy_std_dev": 0.021801399271568482, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.48459852909090906, "avg_duration_per_task_ms": 123998.65090909091, "avg_output_tokens_per_task": 7121.34, "last_run_timestamp": "2026-10-17T11:00:48.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.49828548, "avg_duration_per_task_ms": 348417.68, "avg_output_tokens_per_task": 21971.04, "last_run_timestamp": "2026-10-14T15:16:05.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7866667777777777, "accuracy_std_dev": 0.022662232682876603, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06603865777777777, "avg_duration_per_task_ms": 319761.93111111113, "avg_output_tokens_per_task": 16377.204444444444, "last_run_timestamp": "2026-10-17T20:11:55.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.7866666666666666, "accuracy_std_dev": 0.02880303482081167, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.15989929333333333, "avg_duration_per_task_ms": 77372.58666666667, "avg_output_tokens_per_task": 6353.453333333333, "last_run_timestamp": "2026-10-17T10:25:34.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7866666666666666, "accuracy_std_dev": 0.014401440259764158, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015385413333333335, "avg_duration_per_task_ms": 462000.64666666667, "avg_output_tokens_per_task": 21255.126666666667, "last_run_timestamp": "2026-10-17T11:53:52.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7863944591836735, "accuracy_std_dev": 0.023571561637702326, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.0799341387755102, "avg_duration_per_task_ms": 1168407.112244898, "avg_output_tokens_per_task": 26108.473469387754, "last_run_timestamp": "2026-10-17T20:08:31.778Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7862320000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.029582717391304348, "avg_duration_per_task_ms": 2435445.304347826, "avg_output_tokens_per_task": 108721.58695652174, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7860963903743317, "accuracy_std_dev": 0.025611788150185825, "run_count": 4, "total_questions": 187, "avg_cost_per_task_usd": 0.0982251550802139, "avg_duration_per_task_ms": 69951.20320855615, "avg_output_tokens_per_task": 7732.208556149732, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7860360641891891, "accuracy_std_dev": 0.025554613476174253, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.09440423986486486, "avg_duration_per_task_ms": 381879.5878378378, "avg_output_tokens_per_task": 16484.91891891892, "last_run_timestamp": "2026-10-10T11:07:21.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7859532642140468, "accuracy_std_dev": 0.022060806735148097, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028781180602006688, "avg_duration_per_task_ms": 346700.6989966555, "avg_output_tokens_per_task": 11874.802675585284, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.7855944623115578, "accuracy_std_dev": 0.024102213207504365, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.24837878391959797, "avg_duration_per_task_ms": 56668.633165829146, "avg_output_tokens_per_task": 4728.195979899498, "last_run_timestamp": "2026-10-17T11:02:38.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.7855555833333333, "accuracy_std_dev": 0.01921298590128705, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.18381405, "avg_duration_per_task_ms": 65853.755, "avg_output_tokens_per_task": 6393.828333333333, "last_run_timestamp": "2026-10-17T11:10:53.414Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.7853332000000001, "accuracy_std_dev": 0.019505181079907966, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.070760672, "avg_duration_per_task_ms": 60914.488, "avg_output_tokens_per_task": 12425.428, "last_run_timestamp": "2026-10-17T11:22:17.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.785, "accuracy_std_dev": 0.012360308339600598, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1797631525, "avg_duration_per_task_ms": 69465.2375, "avg_output_tokens_per_task": 6280.4925, "last_run_timestamp": "2026-10-17T10:36:28.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7844445, "accuracy_std_dev": 0.008314898130263933, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08145889999999999, "avg_duration_per_task_ms": 40269.02, "avg_output_tokens_per_task": 4762.626666666667, "last_run_timestamp": "2026-10-17T11:17:26.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.7844443333333334, "accuracy_std_dev": 0.006285236475706792, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.037741833333333336, "avg_duration_per_task_ms": 258684.46, "avg_output_tokens_per_task": 13025.313333333334, "last_run_timestamp": "2026-10-17T11:00:49.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.7838094285714285, "accuracy_std_dev": 0.017036561992685377, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0766554, "avg_duration_per_task_ms": 47593.05714285714, "avg_output_tokens_per_task": 4753.951428571429, "last_run_timestamp": "2026-10-17T11:22:27.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7836751845597104, "accuracy_std_dev": 0.029924488404365016, "run_count": 17, "total_questions": 829, "avg_cost_per_task_usd": 0.0797846031363088, "avg_duration_per_task_ms": 80343.72135102534, "avg_output_tokens_per_task": 4214.624849215923, "last_run_timestamp": "2026-10-17T11:16:45.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.01914828103903846, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012174070000000002, "avg_duration_per_task_ms": 629324.83, "avg_output_tokens_per_task": 23420.965, "last_run_timestamp": "2026-10-17T12:15:06.096Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12497997999999999, "avg_duration_per_task_ms": 155609.03, "avg_output_tokens_per_task": 4544.63, "last_run_timestamp": "2026-10-17T11:01:25.869Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7831052054794521, "accuracy_std_dev": 0.017155537965592597, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.14992966438356164, "avg_duration_per_task_ms": 183129.3904109589, "avg_output_tokens_per_task": 16313.164383561643, "last_run_timestamp": "2026-10-15T21:26:28.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7830555, "accuracy_std_dev": 0.023072206170700996, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.132926985, "avg_duration_per_task_ms": 302505.37, "avg_output_tokens_per_task": 19619.503333333334, "last_run_timestamp": "2026-10-17T11:22:27.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.7826666, "accuracy_std_dev": 0.01372745505328646, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.12898250000000003, "avg_duration_per_task_ms": 448868.648, "avg_output_tokens_per_task": 12081.028, "last_run_timestamp": "2026-10-17T19:47:17.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.7826580270270269, "accuracy_std_dev": 0.01733842438695692, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.03042019594594595, "avg_duration_per_task_ms": 91460.18243243243, "avg_output_tokens_per_task": 15686.162162162162, "last_run_timestamp": "2026-10-17T10:23:46.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7818182727272727, "accuracy_std_dev": 0.022489619278441696, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.4780214336363635, "avg_duration_per_task_ms": 107722.86090909092, "avg_output_tokens_per_task": 7240.321818181818, "last_run_timestamp": "2026-10-17T11:08:24.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7811111666666667, "accuracy_std_dev": 0.027057157151831192, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08468537, "avg_duration_per_task_ms": 260135.41333333333, "avg_output_tokens_per_task": 20074.013333333332, "last_run_timestamp": "2026-10-17T11:21:46.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7811111111111111, "accuracy_std_dev": 0.021998793459058637, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18190104222222223, "avg_duration_per_task_ms": 388430.46, "avg_output_tokens_per_task": 28795.273333333334, "last_run_timestamp": "2026-10-17T19:40:07.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.781111, "accuracy_std_dev": 0.030651344804646544, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03906277, "avg_duration_per_task_ms": 319274.30666666664, "avg_output_tokens_per_task": 12696.91, "last_run_timestamp": "2026-10-17T11:08:09.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7808335, "accuracy_std_dev": 0.0240801728035743, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.056226975, "avg_duration_per_task_ms": 913150.26, "avg_output_tokens_per_task": 39116.4, "last_run_timestamp": "2026-10-17T11:47:31.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.78083325, "accuracy_std_dev": 0.0203954004909808, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08186078000000001, "avg_duration_per_task_ms": 137926.375, "avg_output_tokens_per_task": 4672.7925, "last_run_timestamp": "2026-10-17T10:59:13.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7807406666666666, "accuracy_std_dev": 0.02483414725288101, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10385739777777778, "avg_duration_per_task_ms": 312310.43333333335, "avg_output_tokens_per_task": 20848.71111111111, "last_run_timestamp": "2026-10-17T11:10:01.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7806513735632183, "accuracy_std_dev": 0.017034729877517862, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.1535702011494253, "avg_duration_per_task_ms": 329766.867816092, "avg_output_tokens_per_task": 20311.916666666668, "last_run_timestamp": "2026-10-17T12:02:08.368Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7801960294117648, "accuracy_std_dev": 0.024494153043587038, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.548420931764706, "avg_duration_per_task_ms": 113328.19352941176, "avg_output_tokens_per_task": 7234.435882352941, "last_run_timestamp": "2026-10-17T11:01:24.330Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7800268380566802, "accuracy_std_dev": 0.020840856341331072, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.07761835222672064, "avg_duration_per_task_ms": 235647.0890688259, "avg_output_tokens_per_task": 11752.493927125506, "last_run_timestamp": "2026-10-17T19:18:11.122Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7800003333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2757919799999997, "avg_duration_per_task_ms": 102078.39333333333, "avg_output_tokens_per_task": 5685.653333333334, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.78, "accuracy_std_dev": 0.022222200001100003, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4880859666666667, "avg_duration_per_task_ms": 126960.01777777777, "avg_output_tokens_per_task": 8014.38, "last_run_timestamp": "2026-10-17T11:11:34.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.78, "accuracy_std_dev": 0.025385603971542606, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17071506, "avg_duration_per_task_ms": 87188.75, "avg_output_tokens_per_task": 6362.575, "last_run_timestamp": "2026-10-17T10:59:47.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00748126, "avg_duration_per_task_ms": 121223.72, "avg_output_tokens_per_task": 12121.04, "last_run_timestamp": "2026-10-17T15:39:37.383Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1158576, "avg_duration_per_task_ms": 173684.7, "avg_output_tokens_per_task": 11854.88, "last_run_timestamp": "2026-10-17T22:46:23.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028234659999999998, "avg_duration_per_task_ms": 113376.08, "avg_output_tokens_per_task": 11900.56, "last_run_timestamp": "2026-10-17T19:19:16.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7799998, "accuracy_std_dev": 0.01633001326882497, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.055028052, "avg_duration_per_task_ms": 146838.836, "avg_output_tokens_per_task": 4580.112, "last_run_timestamp": "2026-10-17T10:59:26.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.7799996666666668, "accuracy_std_dev": 0.024944382578492918, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05863376, "avg_duration_per_task_ms": 922572.7666666667, "avg_output_tokens_per_task": 40326.753333333334, "last_run_timestamp": "2026-10-17T11:22:08.858Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.77916675, "accuracy_std_dev": 0.024818790485990654, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0730951225, "avg_duration_per_task_ms": 271033.7, "avg_output_tokens_per_task": 14317.8375, "last_run_timestamp": "2026-10-17T11:02:34.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.77916675, "accuracy_std_dev": 0.020121260868481886, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0690324625, "avg_duration_per_task_ms": 39596.8475, "avg_output_tokens_per_task": 4762.57, "last_run_timestamp": "2026-10-17T11:01:13.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7788889999999999, "accuracy_std_dev": 0.0178124938830399, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15423991666666667, "avg_duration_per_task_ms": 293986.99666666664, "avg_output_tokens_per_task": 19979.47, "last_run_timestamp": "2026-10-17T11:22:17.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.7788888333333334, "accuracy_std_dev": 0.019404546369486603, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09862560333333334, "avg_duration_per_task_ms": 369802.0233333333, "avg_output_tokens_per_task": 20634.286666666667, "last_run_timestamp": "2026-10-17T11:38:21.418Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.778383818181818, "accuracy_std_dev": 0.020761880948324426, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.4916623981818182, "avg_duration_per_task_ms": 123938.08424242424, "avg_output_tokens_per_task": 7196.92303030303, "last_run_timestamp": "2026-10-17T11:07:54.507Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7779999000000001, "accuracy_std_dev": 0.04120956253698892, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11152408, "avg_duration_per_task_ms": 189018.608, "avg_output_tokens_per_task": 6916.256, "last_run_timestamp": "2026-10-17T10:59:44.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.777778, "accuracy_std_dev": 0.034995646995971776, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07155335333333333, "avg_duration_per_task_ms": 807590.7333333333, "avg_output_tokens_per_task": 13442.86, "last_run_timestamp": "2026-10-17T11:32:02.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.7777778333333333, "accuracy_std_dev": 0.019499832225061745, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09435254333333333, "avg_duration_per_task_ms": 128876.18666666666, "avg_output_tokens_per_task": 5214.3, "last_run_timestamp": "2026-10-17T11:32:49.835Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.7777776666666667, "accuracy_std_dev": 0.017497908167041606, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08824630666666666, "avg_duration_per_task_ms": 389172.2, "avg_output_tokens_per_task": 14887.42, "last_run_timestamp": "2026-10-17T10:26:01.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7776190714285715, "accuracy_std_dev": 0.022762808810062738, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.49921174, "avg_duration_per_task_ms": 127401.97714285714, "avg_output_tokens_per_task": 8107.18, "last_run_timestamp": "2026-10-17T10:19:21.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7772191809045226, "accuracy_std_dev": 0.015717449830920396, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.08903428140703519, "avg_duration_per_task_ms": 135997.33417085427, "avg_output_tokens_per_task": 4429.972361809046, "last_run_timestamp": "2026-10-13T19:11:39.344Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7771429999999999, "accuracy_std_dev": 0.015474446789096262, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11423808857142857, "avg_duration_per_task_ms": 264332.0714285714, "avg_output_tokens_per_task": 23099.01714285714, "last_run_timestamp": "2026-10-17T10:59:43.838Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.777037, "accuracy_std_dev": 0.02882250302184812, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01873947555555555, "avg_duration_per_task_ms": 295769.78, "avg_output_tokens_per_task": 11621.488888888889, "last_run_timestamp": "2026-10-17T19:51:29.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7769696363636364, "accuracy_std_dev": 0.015919847448291065, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12614458545454543, "avg_duration_per_task_ms": 91476.05272727273, "avg_output_tokens_per_task": 4526.654545454546, "last_run_timestamp": "2026-10-17T11:13:41.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.776825380952381, "accuracy_std_dev": 0.02899504966529581, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.5120268733333334, "avg_duration_per_task_ms": 129027.85523809523, "avg_output_tokens_per_task": 7974.611428571428, "last_run_timestamp": "2026-10-17T11:11:30.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.77666675, "accuracy_std_dev": 0.019720364546685406, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.44522166500000004, "avg_duration_per_task_ms": 128363.55, "avg_output_tokens_per_task": 7253.05, "last_run_timestamp": "2026-10-17T10:59:59.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7762962222222223, "accuracy_std_dev": 0.028993197254903236, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10068874222222222, "avg_duration_per_task_ms": 236706.54, "avg_output_tokens_per_task": 11538.877777777778, "last_run_timestamp": "2026-10-17T19:39:33.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.7758333749999999, "accuracy_std_dev": 0.022592582670743405, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.18276251000000002, "avg_duration_per_task_ms": 750721.8, "avg_output_tokens_per_task": 29533.0075, "last_run_timestamp": "2026-10-17T19:12:23.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.77583325, "accuracy_std_dev": 0.027067136183709917, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021435382500000003, "avg_duration_per_task_ms": 238417.755, "avg_output_tokens_per_task": 12509.2125, "last_run_timestamp": "2026-10-17T19:35:58.094Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7757142142857143, "accuracy_std_dev": 0.01738592688345774, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.43694479428571437, "avg_duration_per_task_ms": 94452.41714285714, "avg_output_tokens_per_task": 5698.648571428572, "last_run_timestamp": "2026-10-17T11:00:50.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.7755556666666668, "accuracy_std_dev": 0.030952277087298266, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11345276666666666, "avg_duration_per_task_ms": 300353.7133333333, "avg_output_tokens_per_task": 20577.506666666668, "last_run_timestamp": "2026-10-17T11:24:14.273Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.7755553333333334, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.007128806666666667, "avg_duration_per_task_ms": 167248.23333333334, "avg_output_tokens_per_task": 17580.666666666668, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.7755098163265306, "accuracy_std_dev": 0.018194500000000002, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.06427333673469388, "avg_duration_per_task_ms": 82291.80612244898, "avg_output_tokens_per_task": 4253.367346938776, "last_run_timestamp": "2026-10-17T11:13:56.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.775245761394102, "accuracy_std_dev": 0.03049371819200363, "run_count": 15, "total_questions": 746, "avg_cost_per_task_usd": 0.03492235924932976, "avg_duration_per_task_ms": 523697.9343163539, "avg_output_tokens_per_task": 12545.84852546917, "last_run_timestamp": "2026-10-17T19:49:18.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7752381428571429, "accuracy_std_dev": 0.024871433587428726, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05666836000000001, "avg_duration_per_task_ms": 135387.59714285715, "avg_output_tokens_per_task": 14986.728571428572, "last_run_timestamp": "2026-10-17T10:59:42.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7752381428571429, "accuracy_std_dev": 0.029106105056345494, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06096673714285714, "avg_duration_per_task_ms": 121563.02571428572, "avg_output_tokens_per_task": 6310.4457142857145, "last_run_timestamp": "2026-10-17T11:15:29.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.775238, "accuracy_std_dev": 0.01807009782565028, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08216295714285714, "avg_duration_per_task_ms": 86031.87428571428, "avg_output_tokens_per_task": 4554.505714285714, "last_run_timestamp": "2026-10-17T11:15:40.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.775, "accuracy_std_dev": 0.015184019378939179, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17930137999999998, "avg_duration_per_task_ms": 76460.92, "avg_output_tokens_per_task": 6444.745, "last_run_timestamp": "2026-10-17T10:27:00.078Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7748484090909091, "accuracy_std_dev": 0.03107060548706196, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.44300329, "avg_duration_per_task_ms": 80178.89, "avg_output_tokens_per_task": 5667.305454545454, "last_run_timestamp": "2026-10-17T11:01:36.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7744445, "accuracy_std_dev": 0.011166655613178608, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.030003983333333338, "avg_duration_per_task_ms": 137881.70333333334, "avg_output_tokens_per_task": 12197.636666666667, "last_run_timestamp": "2026-10-17T19:40:24.035Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.026945723560596418, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21623159142857146, "avg_duration_per_task_ms": 562306.5571428571, "avg_output_tokens_per_task": 12602.945714285714, "last_run_timestamp": "2026-10-10T11:06:43.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7740742222222222, "accuracy_std_dev": 0.03402175734169657, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06915481111111112, "avg_duration_per_task_ms": 253932.57111111112, "avg_output_tokens_per_task": 12007.897777777778, "last_run_timestamp": "2026-10-17T11:00:48.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.774074, "accuracy_std_dev": 0.039433570402217795, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11824472000000001, "avg_duration_per_task_ms": 275582.18888888886, "avg_output_tokens_per_task": 12068.497777777779, "last_run_timestamp": "2026-10-17T21:03:47.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7737493322091062, "accuracy_std_dev": 0.02457644899198327, "run_count": 12, "total_questions": 593, "avg_cost_per_task_usd": 0.05600873693086003, "avg_duration_per_task_ms": 258548.61720067455, "avg_output_tokens_per_task": 11042.026981450254, "last_run_timestamp": "2026-10-17T10:29:45.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.773639194842407, "accuracy_std_dev": 0.03650679077617633, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.1842622492836676, "avg_duration_per_task_ms": 578395.2607449857, "avg_output_tokens_per_task": 27822.154727793695, "last_run_timestamp": "2026-10-17T19:41:38.122Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7733332222222222, "accuracy_std_dev": 0.02131486469202904, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08245232666666666, "avg_duration_per_task_ms": 336783.29777777777, "avg_output_tokens_per_task": 20135.455555555556, "last_run_timestamp": "2026-10-17T11:24:16.298Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11485508, "avg_duration_per_task_ms": 381375.48, "avg_output_tokens_per_task": 14199.1, "last_run_timestamp": "2026-10-13T03:50:19.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7730925662650602, "accuracy_std_dev": 0.018139575154892704, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.06739513253012049, "avg_duration_per_task_ms": 1083762.6305220884, "avg_output_tokens_per_task": 34958.25702811245, "last_run_timestamp": "2026-10-17T11:17:33.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7729629444444445, "accuracy_std_dev": 0.01981045608166506, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.07089291888888888, "avg_duration_per_task_ms": 76354.76444444444, "avg_output_tokens_per_task": 4694.775555555556, "last_run_timestamp": "2026-10-17T11:17:01.185Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7727272727272727, "accuracy_std_dev": 0.026431515541767755, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.47498244545454554, "avg_duration_per_task_ms": 113596.48181818181, "avg_output_tokens_per_task": 7235.563636363637, "last_run_timestamp": "2026-10-17T11:08:22.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7727271818181819, "accuracy_std_dev": 0.0191652424922077, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4342336036363637, "avg_duration_per_task_ms": 98810.9909090909, "avg_output_tokens_per_task": 5628.896363636363, "last_run_timestamp": "2026-10-17T11:12:37.911Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.772476937614679, "accuracy_std_dev": 0.03305459265948225, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.10160879082568808, "avg_duration_per_task_ms": 803326.4128440367, "avg_output_tokens_per_task": 10658.660550458715, "last_run_timestamp": "2026-10-17T19:57:29.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7722222222222223, "accuracy_std_dev": 0.03095202181433487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03593436666666667, "avg_duration_per_task_ms": 446735.1666666667, "avg_output_tokens_per_task": 13815.786666666667, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.7722221666666667, "accuracy_std_dev": 0.014098441384501401, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08663532333333333, "avg_duration_per_task_ms": 61723.59333333333, "avg_output_tokens_per_task": 4868.923333333333, "last_run_timestamp": "2026-10-17T11:23:57.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7722220000000001, "accuracy_std_dev": 0.024770452330683562, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07917609666666667, "avg_duration_per_task_ms": 223600.7, "avg_output_tokens_per_task": 9330.2, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.771929874493927, "accuracy_std_dev": 0.030837182396580943, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.011004716599190284, "avg_duration_per_task_ms": 1072772.7165991904, "avg_output_tokens_per_task": 18798.943319838058, "last_run_timestamp": "2026-10-17T12:03:02.755Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7718518888888889, "accuracy_std_dev": 0.0203131506142337, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04409809333333333, "avg_duration_per_task_ms": 198272.02666666667, "avg_output_tokens_per_task": 13209.757777777777, "last_run_timestamp": "2026-10-17T19:51:22.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7718517777777778, "accuracy_std_dev": 0.037022202956177475, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06489010666666667, "avg_duration_per_task_ms": 307042.8888888889, "avg_output_tokens_per_task": 12648.484444444444, "last_run_timestamp": "2026-10-17T10:41:37.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.771666625, "accuracy_std_dev": 0.02020731097237767, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0087578875, "avg_duration_per_task_ms": 205985.82, "avg_output_tokens_per_task": 4280.365, "last_run_timestamp": "2026-10-17T11:27:39.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7715152727272727, "accuracy_std_dev": 0.020516898902172905, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5359888036363636, "avg_duration_per_task_ms": 175110.11272727273, "avg_output_tokens_per_task": 10646.576363636364, "last_run_timestamp": "2026-10-17T11:13:46.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7714285714285715, "accuracy_std_dev": 0.017715327672120907, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.059177931428571424, "avg_duration_per_task_ms": 49647.705714285716, "avg_output_tokens_per_task": 4699.5028571428575, "last_run_timestamp": "2026-10-17T10:58:43.523Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.7711111666666667, "accuracy_std_dev": 0.025141594085079193, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04609336666666666, "avg_duration_per_task_ms": 106575.85, "avg_output_tokens_per_task": 4990.85, "last_run_timestamp": "2026-10-17T11:02:44.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.771111, "accuracy_std_dev": 0.04085521560339636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4149127733333333, "avg_duration_per_task_ms": 472514.2266666667, "avg_output_tokens_per_task": 36470.113333333335, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.77083325, "accuracy_std_dev": 0.02099910381867521, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06492240499999999, "avg_duration_per_task_ms": 184655.75, "avg_output_tokens_per_task": 10289.75, "last_run_timestamp": "2026-10-17T19:13:29.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.770833, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019129833333333335, "avg_duration_per_task_ms": 643112.25, "avg_output_tokens_per_task": 13416, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7707407777777777, "accuracy_std_dev": 0.023188894510460925, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11877632888888891, "avg_duration_per_task_ms": 486710.77777777775, "avg_output_tokens_per_task": 21345.64, "last_run_timestamp": "2026-10-17T11:37:24.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7704762857142857, "accuracy_std_dev": 0.01419017577374559, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.15732603714285717, "avg_duration_per_task_ms": 351587.40285714285, "avg_output_tokens_per_task": 16725.24, "last_run_timestamp": "2026-10-17T21:41:22.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7700000714285714, "accuracy_std_dev": 0.028701941199817064, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.5494887142857142, "avg_duration_per_task_ms": 184565.79285714286, "avg_output_tokens_per_task": 10546.365714285714, "last_run_timestamp": "2026-10-17T10:59:10.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7700000312500004, "accuracy_std_dev": 0.02798810387298991, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.58152457625, "avg_duration_per_task_ms": 117734.71, "avg_output_tokens_per_task": 8150.94375, "last_run_timestamp": "2026-10-17T11:11:43.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.77, "accuracy_std_dev": 0.024267078752911318, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.044564936000000006, "avg_duration_per_task_ms": 210405.052, "avg_output_tokens_per_task": 9653.816, "last_run_timestamp": "2026-10-17T19:41:23.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.77, "accuracy_std_dev": 0.027888739266353848, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.049744766666666676, "avg_duration_per_task_ms": 93112.96533333333, "avg_output_tokens_per_task": 4363.234666666666, "last_run_timestamp": "2026-10-17T11:14:20.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7697776666666667, "accuracy_std_dev": 0.021342537367634824, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.019707239999999997, "avg_duration_per_task_ms": 239431.496, "avg_output_tokens_per_task": 16872.549333333332, "last_run_timestamp": "2026-10-17T11:12:59.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7696296666666667, "accuracy_std_dev": 0.023749919068867958, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04528427333333334, "avg_duration_per_task_ms": 235347.72222222222, "avg_output_tokens_per_task": 12527.615555555556, "last_run_timestamp": "2026-10-17T19:58:17.795Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7696296666666667, "accuracy_std_dev": 0.02999782525450804, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.15527621777777778, "avg_duration_per_task_ms": 612793.8288888889, "avg_output_tokens_per_task": 12411.853333333333, "last_run_timestamp": "2026-10-17T11:35:37.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.76958325, "accuracy_std_dev": 0.03006639296935201, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11405574499999999, "avg_duration_per_task_ms": 171710.355, "avg_output_tokens_per_task": 9478.4625, "last_run_timestamp": "2026-10-17T21:04:23.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7695238571428571, "accuracy_std_dev": 0.029354336946199627, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12993965428571427, "avg_duration_per_task_ms": 178407.48, "avg_output_tokens_per_task": 8114.634285714285, "last_run_timestamp": "2026-10-17T12:47:59.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7693334, "accuracy_std_dev": 0.024801480242921004, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06763208, "avg_duration_per_task_ms": 1137766.732, "avg_output_tokens_per_task": 63926.756, "last_run_timestamp": "2026-10-17T19:12:07.371Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.76916675, "accuracy_std_dev": 0.020800176950869918, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0561824625, "avg_duration_per_task_ms": 113575.025, "avg_output_tokens_per_task": 4743.9025, "last_run_timestamp": "2026-10-17T11:00:48.270Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7690910909090909, "accuracy_std_dev": 0.0207480154952645, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4942437709090909, "avg_duration_per_task_ms": 129656.94545454545, "avg_output_tokens_per_task": 8155.923636363636, "last_run_timestamp": "2026-10-17T10:36:13.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7688889999999999, "accuracy_std_dev": 0.027576954057892455, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.13352325333333334, "avg_duration_per_task_ms": 270383.85333333333, "avg_output_tokens_per_task": 11724.017777777777, "last_run_timestamp": "2026-10-17T19:36:13.358Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.7688889999999999, "accuracy_std_dev": 0.016629528997138415, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07698892666666665, "avg_duration_per_task_ms": 578204.64, "avg_output_tokens_per_task": 15573.933333333332, "last_run_timestamp": "2026-10-17T11:11:44.961Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7688889333333333, "accuracy_std_dev": 0.020608014743352204, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.018404090666666668, "avg_duration_per_task_ms": 189791.296, "avg_output_tokens_per_task": 18860.122666666666, "last_run_timestamp": "2026-10-17T11:10:46.239Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.021998731726069008, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09783254, "avg_duration_per_task_ms": 102292.53333333334, "avg_output_tokens_per_task": 23933.02, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.7685714285714286, "accuracy_std_dev": 0.024093478197916338, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04910968857142857, "avg_duration_per_task_ms": 125132.30571428571, "avg_output_tokens_per_task": 4665.682857142857, "last_run_timestamp": "2026-10-17T11:17:33.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7684848636363636, "accuracy_std_dev": 0.030230338267279323, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5669869918181819, "avg_duration_per_task_ms": 182291.99818181817, "avg_output_tokens_per_task": 10636.502727272727, "last_run_timestamp": "2026-10-17T11:15:51.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7684848181818182, "accuracy_std_dev": 0.03062860915673504, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.16836050181818182, "avg_duration_per_task_ms": 595336.3309090909, "avg_output_tokens_per_task": 35883.39090909091, "last_run_timestamp": "2026-10-17T11:01:46.079Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7683335, "accuracy_std_dev": 0.025980601739567167, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.028812655, "avg_duration_per_task_ms": 169670.27, "avg_output_tokens_per_task": 16080.47, "last_run_timestamp": "2026-10-17T11:02:18.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.768333375, "accuracy_std_dev": 0.01755948224562374, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.099579235, "avg_duration_per_task_ms": 250087.7225, "avg_output_tokens_per_task": 17572.65, "last_run_timestamp": "2026-10-17T19:37:12.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.768333375, "accuracy_std_dev": 0.03086715037437658, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0819386525, "avg_duration_per_task_ms": 47567.265, "avg_output_tokens_per_task": 4891.2325, "last_run_timestamp": "2026-10-17T10:41:20.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7681481111111111, "accuracy_std_dev": 0.03307724426061735, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.43618789555555554, "avg_duration_per_task_ms": 93061.98222222223, "avg_output_tokens_per_task": 5582.697777777777, "last_run_timestamp": "2026-10-17T11:14:22.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7681481111111111, "accuracy_std_dev": 0.02261391222453041, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027650264444444447, "avg_duration_per_task_ms": 160206.6511111111, "avg_output_tokens_per_task": 9294.16, "last_run_timestamp": "2026-10-17T19:51:53.679Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7681249375000001, "accuracy_std_dev": 0.02039432387322742, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.5345501306250001, "avg_duration_per_task_ms": 120332.5425, "avg_output_tokens_per_task": 8109.15625, "last_run_timestamp": "2026-10-17T11:09:50.077Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7680001999999999, "accuracy_std_dev": 0.012927091279943853, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034976304, "avg_duration_per_task_ms": 249863.756, "avg_output_tokens_per_task": 38297.836, "last_run_timestamp": "2026-10-17T10:47:05.212Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.768, "accuracy_std_dev": 0.029028767342069495, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.011671187999999999, "avg_duration_per_task_ms": 241079.168, "avg_output_tokens_per_task": 6128.846, "last_run_timestamp": "2026-10-17T10:59:30.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7677778333333333, "accuracy_std_dev": 0.024469832777092887, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.22403587333333333, "avg_duration_per_task_ms": 379432.67333333334, "avg_output_tokens_per_task": 17682.066666666666, "last_run_timestamp": "2026-10-17T11:24:46.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.76777775, "accuracy_std_dev": 0.024011215230682656, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09013422333333333, "avg_duration_per_task_ms": 187936.75166666668, "avg_output_tokens_per_task": 11408.701666666666, "last_run_timestamp": "2026-10-17T20:10:05.740Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7677776666666667, "accuracy_std_dev": 0.030408559549720344, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.032006433333333334, "avg_duration_per_task_ms": 190973.69, "avg_output_tokens_per_task": 14850.593333333334, "last_run_timestamp": "2026-10-17T10:41:27.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7676191428571428, "accuracy_std_dev": 0.019657045959063507, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05359846000000001, "avg_duration_per_task_ms": 117300.16571428571, "avg_output_tokens_per_task": 4471.66, "last_run_timestamp": "2026-10-17T11:03:28.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.767500125, "accuracy_std_dev": 0.029521664597535403, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.14441939499999998, "avg_duration_per_task_ms": 322443.2475, "avg_output_tokens_per_task": 20266.705, "last_run_timestamp": "2026-10-17T12:43:23.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7674073333333333, "accuracy_std_dev": 0.025423825815779805, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18668503555555555, "avg_duration_per_task_ms": 177779.4688888889, "avg_output_tokens_per_task": 9629.98, "last_run_timestamp": "2026-10-17T19:18:49.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7671692361809045, "accuracy_std_dev": 0.02443026293350114, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.10522786934673367, "avg_duration_per_task_ms": 46016.98994974874, "avg_output_tokens_per_task": 4282.550251256282, "last_run_timestamp": "2026-10-17T11:11:52.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7671130223214285, "accuracy_std_dev": 0.019417018578276532, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.10946027455357142, "avg_duration_per_task_ms": 324665.375, "avg_output_tokens_per_task": 11865.004464285714, "last_run_timestamp": "2026-10-17T19:43:35.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7670682771084337, "accuracy_std_dev": 0.016486696678231213, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.02956908032128514, "avg_duration_per_task_ms": 107195.45381526105, "avg_output_tokens_per_task": 18350.590361445782, "last_run_timestamp": "2026-10-17T11:00:50.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025223939999999997, "avg_duration_per_task_ms": 165137.1, "avg_output_tokens_per_task": 14240.82, "last_run_timestamp": "2026-09-15T00:00:05.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.010828599999999999, "avg_duration_per_task_ms": 269370.42, "avg_output_tokens_per_task": 14956.26, "last_run_timestamp": "2026-10-17T15:38:31.686Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7666667142857143, "accuracy_std_dev": 0.019189911372640464, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06833657142857143, "avg_duration_per_task_ms": 120543.64, "avg_output_tokens_per_task": 4633.334285714286, "last_run_timestamp": "2026-10-17T10:58:54.174Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.014401440259764175, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3460653, "avg_duration_per_task_ms": 170899.55333333334, "avg_output_tokens_per_task": 12611.153333333334, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.766666625, "accuracy_std_dev": 0.022360828847213497, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.010823125, "avg_duration_per_task_ms": 1144392.885, "avg_output_tokens_per_task": 13299.99, "last_run_timestamp": "2026-10-17T19:43:24.819Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7666665714285714, "accuracy_std_dev": 0.041861494979983496, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009251074285714285, "avg_duration_per_task_ms": 197957.5657142857, "avg_output_tokens_per_task": 15755.4, "last_run_timestamp": "2026-10-17T19:15:12.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.27949224, "avg_duration_per_task_ms": 363191.33, "avg_output_tokens_per_task": 35460.86, "last_run_timestamp": "2026-10-15T21:31:19.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7666665, "accuracy_std_dev": 0.05270466983342177, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.066754105, "avg_duration_per_task_ms": 1006890.58, "avg_output_tokens_per_task": 42879.28, "last_run_timestamp": "2026-10-17T11:45:34.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.7666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01352679, "avg_duration_per_task_ms": 90848.6, "avg_output_tokens_per_task": 4507.38, "last_run_timestamp": "2026-10-10T11:18:50.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7663313768844222, "accuracy_std_dev": 0.021610498297991634, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.06149351256281408, "avg_duration_per_task_ms": 193004.91959798994, "avg_output_tokens_per_task": 7834.050251256282, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7660606363636364, "accuracy_std_dev": 0.02228458723765413, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.6243563309090909, "avg_duration_per_task_ms": 72757.64666666667, "avg_output_tokens_per_task": 5695.7472727272725, "last_run_timestamp": "2026-10-17T11:03:34.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7660606363636363, "accuracy_std_dev": 0.024029451627203914, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4185906163636364, "avg_duration_per_task_ms": 96036.28, "avg_output_tokens_per_task": 5622.494545454545, "last_run_timestamp": "2026-10-17T10:59:55.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.76583325, "accuracy_std_dev": 0.03340613508305772, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0064167, "avg_duration_per_task_ms": 121615.31, "avg_output_tokens_per_task": 4277.8575, "last_run_timestamp": "2026-10-17T11:37:18.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.765625, "accuracy_std_dev": 0.00520799999999999, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.06623564583333334, "avg_duration_per_task_ms": 151056.27083333334, "avg_output_tokens_per_task": 10137.447916666666, "last_run_timestamp": "2026-10-17T19:11:05.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7656249866071428, "accuracy_std_dev": 0.0414301585804351, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.028515656250000004, "avg_duration_per_task_ms": 494142.7745535714, "avg_output_tokens_per_task": 12506.10044642857, "last_run_timestamp": "2026-10-17T19:35:17.391Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7655824146341464, "accuracy_std_dev": 0.030599644599243303, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.07519595528455283, "avg_duration_per_task_ms": 226711.57317073172, "avg_output_tokens_per_task": 10093.731707317073, "last_run_timestamp": "2026-10-10T19:17:08.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.7653333999999999, "accuracy_std_dev": 0.0185713806121139, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.020849908, "avg_duration_per_task_ms": 114504.516, "avg_output_tokens_per_task": 8739.532, "last_run_timestamp": "2026-10-17T19:15:12.182Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.7653332, "accuracy_std_dev": 0.014236029409916211, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05721586, "avg_duration_per_task_ms": 73979.412, "avg_output_tokens_per_task": 4468.456, "last_run_timestamp": "2026-10-17T10:59:41.246Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7652466315789472, "accuracy_std_dev": 0.0204935205903598, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.01140678947368421, "avg_duration_per_task_ms": 287101.95238095237, "avg_output_tokens_per_task": 12923.513784461153, "last_run_timestamp": "2026-10-17T19:01:11.679Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.7651005838926175, "accuracy_std_dev": 0.022563981273013175, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.037864174496644296, "avg_duration_per_task_ms": 157541.35570469798, "avg_output_tokens_per_task": 25184.140939597317, "last_run_timestamp": "2026-10-17T11:16:26.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7650601887550201, "accuracy_std_dev": 0.019886756371012337, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.014014425702811244, "avg_duration_per_task_ms": 335075.9437751004, "avg_output_tokens_per_task": 16676.168674698794, "last_run_timestamp": "2026-10-17T11:01:25.260Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.765, "accuracy_std_dev": 0.03840572873934304, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08522504, "avg_duration_per_task_ms": 558480.23, "avg_output_tokens_per_task": 11022.365, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.765, "accuracy_std_dev": 0.01997680136224682, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.019566853333333332, "avg_duration_per_task_ms": 556627.0233333333, "avg_output_tokens_per_task": 13187.933333333332, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7649999375000001, "accuracy_std_dev": 0.017559488177580636, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.00967452625, "avg_duration_per_task_ms": 342838.47625, "avg_output_tokens_per_task": 14737.495, "last_run_timestamp": "2026-10-17T11:21:51.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.7649122421052632, "accuracy_std_dev": 0.0039184999999999914, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.25060463157894736, "avg_duration_per_task_ms": 358896.81052631576, "avg_output_tokens_per_task": 6743.5684210526315, "last_run_timestamp": "2026-10-10T11:20:06.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.764846632183908, "accuracy_std_dev": 0.017727680655646182, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.01970142528735632, "avg_duration_per_task_ms": 118406.38793103448, "avg_output_tokens_per_task": 14534.735632183909, "last_run_timestamp": "2026-10-17T11:01:24.414Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7647619999999999, "accuracy_std_dev": 0.014998270500294357, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.17264639428571427, "avg_duration_per_task_ms": 205532.82, "avg_output_tokens_per_task": 11242.805714285714, "last_run_timestamp": "2026-10-17T19:36:29.795Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.02424157229270045, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01896127, "avg_duration_per_task_ms": 326667.0966666667, "avg_output_tokens_per_task": 14093.796666666667, "last_run_timestamp": "2026-10-17T10:27:00.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02870791333333333, "avg_duration_per_task_ms": 1126711.7933333332, "avg_output_tokens_per_task": 60958.193333333336, "last_run_timestamp": "2026-10-10T12:23:30.561Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7638095, "accuracy_std_dev": 0.02710519944256874, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.5202140842857143, "avg_duration_per_task_ms": 218955.78857142857, "avg_output_tokens_per_task": 10161.987142857142, "last_run_timestamp": "2026-10-17T11:16:23.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7637036666666667, "accuracy_std_dev": 0.02456770717832659, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5482636911111111, "avg_duration_per_task_ms": 180041.81555555554, "avg_output_tokens_per_task": 10794.96, "last_run_timestamp": "2026-10-17T11:22:39.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.7636362727272727, "accuracy_std_dev": 0.0307959485974459, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5473690363636363, "avg_duration_per_task_ms": 191557.85636363635, "avg_output_tokens_per_task": 10669.91090909091, "last_run_timestamp": "2026-10-17T11:08:54.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7633818649635037, "accuracy_std_dev": 0.03905021498570464, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.10277552189781022, "avg_duration_per_task_ms": 338883.84854014596, "avg_output_tokens_per_task": 17120.268248175184, "last_run_timestamp": "2026-10-18T01:42:21.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01259681, "avg_duration_per_task_ms": 657803.07, "avg_output_tokens_per_task": 25182.87, "last_run_timestamp": "2026-10-03T08:05:49.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.0074535971978300555, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.030563419999999997, "avg_duration_per_task_ms": 58422.895, "avg_output_tokens_per_task": 4829.025, "last_run_timestamp": "2026-10-17T12:23:26.697Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7630522329317269, "accuracy_std_dev": 0.017240119992621882, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.16262659437751, "avg_duration_per_task_ms": 831035.3815261044, "avg_output_tokens_per_task": 31809.98795180723, "last_run_timestamp": "2026-10-10T18:46:35.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7629022732240437, "accuracy_std_dev": 0.028478290118661355, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.01388206375227687, "avg_duration_per_task_ms": 226746.868852459, "avg_output_tokens_per_task": 15915.701275045538, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7627176144578314, "accuracy_std_dev": 0.038350734650199336, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.19653478714859438, "avg_duration_per_task_ms": 478839.8875502008, "avg_output_tokens_per_task": 27836.25702811245, "last_run_timestamp": "2026-10-17T19:45:29.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7626666, "accuracy_std_dev": 0.03227647784749755, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.146569104, "avg_duration_per_task_ms": 373419.58, "avg_output_tokens_per_task": 24831.152, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.762500125, "accuracy_std_dev": 0.01823820585637127, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014795629999999999, "avg_duration_per_task_ms": 251230.48, "avg_output_tokens_per_task": 15189.695, "last_run_timestamp": "2026-10-17T19:16:55.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.7625, "accuracy_std_dev": 0.027271719174815526, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08931772499999999, "avg_duration_per_task_ms": 299563.8, "avg_output_tokens_per_task": 12112.5275, "last_run_timestamp": "2026-10-17T11:07:54.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.762499875, "accuracy_std_dev": 0.030900828154102528, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.00903259, "avg_duration_per_task_ms": 197139.9675, "avg_output_tokens_per_task": 4490.1425, "last_run_timestamp": "2026-10-17T11:12:20.710Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7624549738058551, "accuracy_std_dev": 0.028041605618591807, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.08517802465331278, "avg_duration_per_task_ms": 145563.85516178736, "avg_output_tokens_per_task": 12573.135593220339, "last_run_timestamp": "2026-10-17T19:06:47.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7624242727272728, "accuracy_std_dev": 0.022611815825484268, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.048589845454545456, "avg_duration_per_task_ms": 136956.19272727272, "avg_output_tokens_per_task": 4598.596363636364, "last_run_timestamp": "2026-10-16T01:58:40.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.039377702534415165, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13604349999999998, "avg_duration_per_task_ms": 1230822.4133333333, "avg_output_tokens_per_task": 30337.52, "last_run_timestamp": "2026-10-17T11:00:12.961Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7622222222222222, "accuracy_std_dev": 0.03325913222886738, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04489894888888889, "avg_duration_per_task_ms": 204045.26444444444, "avg_output_tokens_per_task": 9550.304444444444, "last_run_timestamp": "2026-10-17T19:42:12.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7622101098901098, "accuracy_std_dev": 0.024541643374153495, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.2003756904761905, "avg_duration_per_task_ms": 486645.7838827839, "avg_output_tokens_per_task": 20463.4358974359, "last_run_timestamp": "2026-10-17T11:13:24.327Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7617447986577182, "accuracy_std_dev": 0.00897518183195566, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.0072220302013422805, "avg_duration_per_task_ms": 277237.70805369125, "avg_output_tokens_per_task": 12532.48322147651, "last_run_timestamp": "2026-10-17T19:46:31.388Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.76166675, "accuracy_std_dev": 0.011902275693643652, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.080519975, "avg_duration_per_task_ms": 148683.485, "avg_output_tokens_per_task": 8604.13, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7616665, "accuracy_std_dev": 0.01364229534389284, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03678678, "avg_duration_per_task_ms": 78488.155, "avg_output_tokens_per_task": 4646.9025, "last_run_timestamp": "2026-10-17T11:01:16.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.7611212248628885, "accuracy_std_dev": 0.04241023232664495, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.07025861060329068, "avg_duration_per_task_ms": 209956.5265082267, "avg_output_tokens_per_task": 12430.747714808043, "last_run_timestamp": "2026-10-18T01:41:51.190Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.7611110833333333, "accuracy_std_dev": 0.019404866071247587, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.006715375000000001, "avg_duration_per_task_ms": 256863.22, "avg_output_tokens_per_task": 15279.346666666666, "last_run_timestamp": "2026-10-17T11:16:57.833Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.029261543259342417, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.025921108571428573, "avg_duration_per_task_ms": 269473.77714285714, "avg_output_tokens_per_task": 13250.331428571428, "last_run_timestamp": "2026-10-17T19:03:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7609090909090909, "accuracy_std_dev": 0.01608635842996141, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3294631345454546, "avg_duration_per_task_ms": 293594.13636363635, "avg_output_tokens_per_task": 10568.594545454545, "last_run_timestamp": "2026-10-17T11:17:24.553Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7607481658291457, "accuracy_std_dev": 0.027232714686338207, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.061689276381909554, "avg_duration_per_task_ms": 257829.17755443885, "avg_output_tokens_per_task": 11960.412060301507, "last_run_timestamp": "2026-10-17T10:59:41.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.76046883919598, "accuracy_std_dev": 0.03279282531286379, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.07501306030150753, "avg_duration_per_task_ms": 437868.7336683417, "avg_output_tokens_per_task": 62802.35175879397, "last_run_timestamp": "2026-10-17T19:45:57.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7600536219839142, "accuracy_std_dev": 0.024783783422409292, "run_count": 15, "total_questions": 746, "avg_cost_per_task_usd": 0.020043907506702412, "avg_duration_per_task_ms": 441729.95174262737, "avg_output_tokens_per_task": 16926.498659517427, "last_run_timestamp": "2026-10-17T10:58:50.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0113985, "avg_duration_per_task_ms": 89553.33, "avg_output_tokens_per_task": 10995.52, "last_run_timestamp": "2026-10-17T11:01:12.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.76, "accuracy_std_dev": 0.038559978206331065, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1978464090909091, "avg_duration_per_task_ms": 103130.56181818181, "avg_output_tokens_per_task": 7676.363636363636, "last_run_timestamp": "2026-10-17T11:00:48.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.76, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11891346, "avg_duration_per_task_ms": 226293.89, "avg_output_tokens_per_task": 28257, "last_run_timestamp": "2026-10-17T10:59:28.226Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019761673333333334, "avg_duration_per_task_ms": 291193.25333333336, "avg_output_tokens_per_task": 17811.393333333333, "last_run_timestamp": "2026-10-10T11:23:23.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.019457739999999998, "avg_duration_per_task_ms": 87744.74, "avg_output_tokens_per_task": 11443.04, "last_run_timestamp": "2026-10-17T19:53:56.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0448255, "avg_duration_per_task_ms": 126685.96, "avg_output_tokens_per_task": 15758.14, "last_run_timestamp": "2026-10-17T10:19:11.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012781093333333333, "avg_duration_per_task_ms": 706143.8066666666, "avg_output_tokens_per_task": 24864.26, "last_run_timestamp": "2026-10-17T12:54:32.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.025531142930416043, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.013140583333333336, "avg_duration_per_task_ms": 222263.59333333332, "avg_output_tokens_per_task": 5092.44, "last_run_timestamp": "2026-10-17T11:12:55.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7597916562500001, "accuracy_std_dev": 0.03048148907596357, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.20304805999999995, "avg_duration_per_task_ms": 119095.471875, "avg_output_tokens_per_task": 7815.745625, "last_run_timestamp": "2026-10-17T11:12:58.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7596329412844037, "accuracy_std_dev": 0.03240988921571945, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.1936376660550459, "avg_duration_per_task_ms": 554065.5724770642, "avg_output_tokens_per_task": 27060.469724770643, "last_run_timestamp": "2026-10-17T11:01:59.994Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7596315477386935, "accuracy_std_dev": 0.034079836369912346, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.011217208542713568, "avg_duration_per_task_ms": 145495.03015075377, "avg_output_tokens_per_task": 13403.010050251256, "last_run_timestamp": "2026-10-17T19:18:31.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.759408620967742, "accuracy_std_dev": 0.023253272624729635, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.012160826612903226, "avg_duration_per_task_ms": 309190.9919354839, "avg_output_tokens_per_task": 4125.725806451613, "last_run_timestamp": "2026-10-17T11:47:18.976Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7593333, "accuracy_std_dev": 0.016452043441773433, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.083344808, "avg_duration_per_task_ms": 106115.21, "avg_output_tokens_per_task": 23741.31, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7592592222222222, "accuracy_std_dev": 0.03086336097769501, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10491929333333334, "avg_duration_per_task_ms": 277405.59777777776, "avg_output_tokens_per_task": 12223.173333333334, "last_run_timestamp": "2026-10-17T11:16:04.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7590477142857143, "accuracy_std_dev": 0.013997181448055748, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.24529245999999996, "avg_duration_per_task_ms": 151053.55142857143, "avg_output_tokens_per_task": 23224.54, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7590475714285715, "accuracy_std_dev": 0.036934642282423845, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09028300285714286, "avg_duration_per_task_ms": 562815.2714285714, "avg_output_tokens_per_task": 12727.568571428572, "last_run_timestamp": "2026-10-17T11:01:50.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.7590475714285715, "accuracy_std_dev": 0.019331153995980878, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007369005714285714, "avg_duration_per_task_ms": 125780.48857142858, "avg_output_tokens_per_task": 4298.177142857143, "last_run_timestamp": "2026-10-17T11:22:24.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7588888888888888, "accuracy_std_dev": 0.04049087993732375, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.15803105555555555, "avg_duration_per_task_ms": 321070.18888888886, "avg_output_tokens_per_task": 22494.742222222223, "last_run_timestamp": "2026-10-17T10:25:46.397Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7588888333333333, "accuracy_std_dev": 0.010482195371464689, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.061315760000000004, "avg_duration_per_task_ms": 281940.7266666667, "avg_output_tokens_per_task": 13996.163333333334, "last_run_timestamp": "2026-10-17T10:26:47.850Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7587879090909091, "accuracy_std_dev": 0.029103513053970746, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.20085546545454547, "avg_duration_per_task_ms": 125448.63818181818, "avg_output_tokens_per_task": 7874.443636363636, "last_run_timestamp": "2026-10-17T11:03:07.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7586728009049774, "accuracy_std_dev": 0.02217328649078593, "run_count": 9, "total_questions": 442, "avg_cost_per_task_usd": 0.025404156108597285, "avg_duration_per_task_ms": 400240.5045248869, "avg_output_tokens_per_task": 12533.12443438914, "last_run_timestamp": "2026-10-17T19:19:50.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7585186666666667, "accuracy_std_dev": 0.031234283510982678, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08685644444444443, "avg_duration_per_task_ms": 69302.62222222223, "avg_output_tokens_per_task": 12174.302222222223, "last_run_timestamp": "2026-10-17T19:41:24.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.7583335, "accuracy_std_dev": 0.023273625990592902, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.28355265, "avg_duration_per_task_ms": 276032.65, "avg_output_tokens_per_task": 10176.66, "last_run_timestamp": "2026-10-17T11:01:26.330Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.758333375, "accuracy_std_dev": 0.023273652844673413, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.007575310000000001, "avg_duration_per_task_ms": 189684.715, "avg_output_tokens_per_task": 4442.28, "last_run_timestamp": "2026-10-17T11:47:44.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7583333333333333, "accuracy_std_dev": 0.021963715871611736, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.022033749999999998, "avg_duration_per_task_ms": 126674.91333333333, "avg_output_tokens_per_task": 17337.105, "last_run_timestamp": "2026-10-17T11:08:52.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.02074966532712037, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02415933, "avg_duration_per_task_ms": 139345.795, "avg_output_tokens_per_task": 13798.13, "last_run_timestamp": "2026-10-17T19:37:45.712Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.40697005500000005, "avg_duration_per_task_ms": 402356.405, "avg_output_tokens_per_task": 35595.7, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7583331666666667, "accuracy_std_dev": 0.013844473427047428, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09591996333333334, "avg_duration_per_task_ms": 214673.9, "avg_output_tokens_per_task": 9689.54, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.7582582837837838, "accuracy_std_dev": 0.01616414926104476, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.03309702927927928, "avg_duration_per_task_ms": 172017.87837837837, "avg_output_tokens_per_task": 16666.704954954956, "last_run_timestamp": "2026-10-17T11:14:13.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.758189028901734, "accuracy_std_dev": 0.033845822723590796, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.057105187861271674, "avg_duration_per_task_ms": 394678.4624277457, "avg_output_tokens_per_task": 12961.20520231214, "last_run_timestamp": "2026-10-17T11:07:54.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7579964141414142, "accuracy_std_dev": 0.023533015514113357, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20713992929292932, "avg_duration_per_task_ms": 664979.7727272727, "avg_output_tokens_per_task": 53099.67929292929, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012770106666666666, "avg_duration_per_task_ms": 94420.36666666667, "avg_output_tokens_per_task": 11193.713333333333, "last_run_timestamp": "2026-10-17T10:59:45.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.04629468145358481, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12420428666666666, "avg_duration_per_task_ms": 116279.86, "avg_output_tokens_per_task": 19029.046666666665, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.35916456, "avg_duration_per_task_ms": 165675.65333333332, "avg_output_tokens_per_task": 12758.866666666667, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7576190714285714, "accuracy_std_dev": 0.02276274605096257, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.028185067142857137, "avg_duration_per_task_ms": 173417.5, "avg_output_tokens_per_task": 17841.255714285715, "last_run_timestamp": "2026-10-17T11:01:48.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.7575758181818182, "accuracy_std_dev": 0.03306223866318624, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.020269487272727277, "avg_duration_per_task_ms": 191902.98727272727, "avg_output_tokens_per_task": 14622.787272727273, "last_run_timestamp": "2026-10-17T11:03:04.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.75750025, "accuracy_std_dev": 0.017618680900893247, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.14844741250000001, "avg_duration_per_task_ms": 168776.3125, "avg_output_tokens_per_task": 20104.8775, "last_run_timestamp": "2026-10-17T11:13:05.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7575, "accuracy_std_dev": 0.024022628223198242, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.069431845, "avg_duration_per_task_ms": 168602.24, "avg_output_tokens_per_task": 4679.8025, "last_run_timestamp": "2026-10-17T11:23:18.111Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7574072222222222, "accuracy_std_dev": 0.030298005023351983, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015441244444444443, "avg_duration_per_task_ms": 72124.47777777778, "avg_output_tokens_per_task": 4310.673333333333, "last_run_timestamp": "2026-10-17T11:14:15.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.05359938971518238, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023817484, "avg_duration_per_task_ms": 360909.7, "avg_output_tokens_per_task": 13950.552, "last_run_timestamp": "2026-10-17T10:59:33.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7573333, "accuracy_std_dev": 0.021333193750819417, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.017451074, "avg_duration_per_task_ms": 122650.494, "avg_output_tokens_per_task": 4554.818, "last_run_timestamp": "2026-10-17T11:17:33.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.757143, "accuracy_std_dev": 0.03282614272627407, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014302074285714289, "avg_duration_per_task_ms": 167054.57142857142, "avg_output_tokens_per_task": 14974.468571428572, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7571428571428571, "accuracy_std_dev": 0.017405469989374624, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02240780857142857, "avg_duration_per_task_ms": 143693.9342857143, "avg_output_tokens_per_task": 16873.04, "last_run_timestamp": "2026-10-17T11:13:54.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.7569232307692307, "accuracy_std_dev": 0.022282632551121173, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.25500098, "avg_duration_per_task_ms": 282733.35846153845, "avg_output_tokens_per_task": 10086.226153846153, "last_run_timestamp": "2026-10-17T11:10:01.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.7566667499999999, "accuracy_std_dev": 0.026874078797002546, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07064113500000001, "avg_duration_per_task_ms": 324361.97, "avg_output_tokens_per_task": 14598.985, "last_run_timestamp": "2026-10-17T11:22:26.174Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7566667, "accuracy_std_dev": 0.036301371263493636, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.090532718, "avg_duration_per_task_ms": 236615.216, "avg_output_tokens_per_task": 12357.186, "last_run_timestamp": "2026-10-17T10:27:20.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.02285220631337136, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023255454999999998, "avg_duration_per_task_ms": 118711.59, "avg_output_tokens_per_task": 15018.058333333332, "last_run_timestamp": "2026-10-17T10:59:43.836Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.756666625, "accuracy_std_dev": 0.028867614496774323, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.022124855000000002, "avg_duration_per_task_ms": 493245.72, "avg_output_tokens_per_task": 14533.4, "last_run_timestamp": "2026-10-17T19:02:12.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7565656363636365, "accuracy_std_dev": 0.03454844880508337, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.22266426666666672, "avg_duration_per_task_ms": 96431.12363636364, "avg_output_tokens_per_task": 7566.764242424242, "last_run_timestamp": "2026-10-17T11:22:38.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7564362073170733, "accuracy_std_dev": 0.024511456002449167, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.014174674796747964, "avg_duration_per_task_ms": 53986.221544715445, "avg_output_tokens_per_task": 4039.2662601626016, "last_run_timestamp": "2026-10-17T11:16:33.462Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7563351081871345, "accuracy_std_dev": 0.013338594901982123, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.033182684210526316, "avg_duration_per_task_ms": 2146298.769005848, "avg_output_tokens_per_task": 50213.23391812865, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7562961111111112, "accuracy_std_dev": 0.02650155778836174, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013569731111111111, "avg_duration_per_task_ms": 218757.14444444445, "avg_output_tokens_per_task": 14705.06, "last_run_timestamp": "2026-10-17T19:58:35.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7561905714285714, "accuracy_std_dev": 0.01326515904656301, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.19363898571428573, "avg_duration_per_task_ms": 310234.1457142857, "avg_output_tokens_per_task": 12005.822857142857, "last_run_timestamp": "2026-10-17T10:45:58.767Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7561904285714286, "accuracy_std_dev": 0.019633948342873183, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.026500512857142854, "avg_duration_per_task_ms": 291184.5557142857, "avg_output_tokens_per_task": 17123.52857142857, "last_run_timestamp": "2026-10-17T11:00:59.508Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.756, "accuracy_std_dev": 0.024073867807230306, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.351000588, "avg_duration_per_task_ms": 102672.34, "avg_output_tokens_per_task": 7151.472, "last_run_timestamp": "2026-10-17T11:07:54.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7558921212121212, "accuracy_std_dev": 0.026509964125541553, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.0215872904040404, "avg_duration_per_task_ms": 248439.8383838384, "avg_output_tokens_per_task": 4242.020202020202, "last_run_timestamp": "2026-10-17T11:01:48.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.75583325, "accuracy_std_dev": 0.016308889023704216, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09814360999999999, "avg_duration_per_task_ms": 308482.07, "avg_output_tokens_per_task": 12874.305, "last_run_timestamp": "2026-10-17T19:56:39.413Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.75583325, "accuracy_std_dev": 0.017300429508757886, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17775563000000003, "avg_duration_per_task_ms": 323022.9725, "avg_output_tokens_per_task": 10875.7, "last_run_timestamp": "2026-10-17T10:27:08.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7557576363636364, "accuracy_std_dev": 0.027008915986704473, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5367455872727274, "avg_duration_per_task_ms": 190847.75454545455, "avg_output_tokens_per_task": 10370.42909090909, "last_run_timestamp": "2026-10-17T10:58:42.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7557079178082191, "accuracy_std_dev": 0.01607886717264481, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.2608585205479452, "avg_duration_per_task_ms": 125025.47260273973, "avg_output_tokens_per_task": 7152.054794520548, "last_run_timestamp": "2026-10-03T08:05:09.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7555555555555555, "accuracy_std_dev": 0.016329992101182772, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06037513111111111, "avg_duration_per_task_ms": 215849.39333333334, "avg_output_tokens_per_task": 10590.155555555555, "last_run_timestamp": "2026-10-17T19:53:48.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7555553333333334, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010344493333333333, "avg_duration_per_task_ms": 251680.81333333332, "avg_output_tokens_per_task": 16163.846666666666, "last_run_timestamp": "2026-10-17T11:43:44.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7550001666666666, "accuracy_std_dev": 0.026925751821237763, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.014109759999999997, "avg_duration_per_task_ms": 255723.91333333333, "avg_output_tokens_per_task": 16860.226666666666, "last_run_timestamp": "2026-10-17T11:01:23.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.755, "accuracy_std_dev": 0.019649908116833523, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.33917411000000003, "avg_duration_per_task_ms": 125094.895, "avg_output_tokens_per_task": 7173.1, "last_run_timestamp": "2026-10-17T10:36:11.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.755, "accuracy_std_dev": 0.021666999999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03593898, "avg_duration_per_task_ms": 72072.58, "avg_output_tokens_per_task": 16785.94, "last_run_timestamp": "2026-10-17T11:01:14.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.7548148888888889, "accuracy_std_dev": 0.027716145949987135, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.33142048444444444, "avg_duration_per_task_ms": 603214.5822222222, "avg_output_tokens_per_task": 27052.68, "last_run_timestamp": "2026-10-17T19:11:02.176Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7546065175879397, "accuracy_std_dev": 0.07084324947628658, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.25537817085427134, "avg_duration_per_task_ms": 457516.3442211055, "avg_output_tokens_per_task": 29053.731155778896, "last_run_timestamp": "2026-10-17T19:15:48.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7546032380952381, "accuracy_std_dev": 0.03353078090196666, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.20101115333333333, "avg_duration_per_task_ms": 102086.14857142857, "avg_output_tokens_per_task": 7692.0952380952385, "last_run_timestamp": "2026-10-17T11:23:18.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7545454545454545, "accuracy_std_dev": 0.020659486891294522, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.34660061999999997, "avg_duration_per_task_ms": 220288.40363636363, "avg_output_tokens_per_task": 11981.236363636364, "last_run_timestamp": "2026-10-17T11:15:56.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7544342825688073, "accuracy_std_dev": 0.02504935915479907, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.1000418, "avg_duration_per_task_ms": 796849.0642201835, "avg_output_tokens_per_task": 21296.05137614679, "last_run_timestamp": "2026-10-10T11:45:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.75416675, "accuracy_std_dev": 0.023555451717702226, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.37240653, "avg_duration_per_task_ms": 164105.02375, "avg_output_tokens_per_task": 11891.904375, "last_run_timestamp": "2026-10-17T11:22:25.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.754166625, "accuracy_std_dev": 0.01898479106375351, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.00794965, "avg_duration_per_task_ms": 155474.15, "avg_output_tokens_per_task": 13162.275, "last_run_timestamp": "2026-10-17T19:15:01.096Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7539394545454545, "accuracy_std_dev": 0.027185124626810275, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4974660309090909, "avg_duration_per_task_ms": 195466.43818181817, "avg_output_tokens_per_task": 10115.303636363637, "last_run_timestamp": "2026-10-17T11:26:23.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.7538460512820513, "accuracy_std_dev": 0.013548520491182818, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.0682567846153846, "avg_duration_per_task_ms": 219433.69743589743, "avg_output_tokens_per_task": 7166.764102564102, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7537880151515151, "accuracy_std_dev": 0.02397695833731574, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.01387228787878788, "avg_duration_per_task_ms": 200605.87121212122, "avg_output_tokens_per_task": 4857.164141414141, "last_run_timestamp": "2026-10-17T11:28:07.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7537594536340853, "accuracy_std_dev": 0.022172530138312475, "run_count": 16, "total_questions": 798, "avg_cost_per_task_usd": 0.007178401002506266, "avg_duration_per_task_ms": 202699.93609022556, "avg_output_tokens_per_task": 13304.931077694235, "last_run_timestamp": "2026-10-17T11:00:48.272Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.7534247123287672, "accuracy_std_dev": 0.009625530542376443, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.033538157534246577, "avg_duration_per_task_ms": 519040.26712328766, "avg_output_tokens_per_task": 11776.294520547945, "last_run_timestamp": "2026-10-17T11:01:17.325Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.753333375, "accuracy_std_dev": 0.03349946931645897, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.007990629999999999, "avg_duration_per_task_ms": 168858.505, "avg_output_tokens_per_task": 4418.365, "last_run_timestamp": "2026-10-17T11:15:43.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.027755466596690478, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02379494888888889, "avg_duration_per_task_ms": 218862.64444444445, "avg_output_tokens_per_task": 13387.833333333334, "last_run_timestamp": "2026-10-17T19:57:48.068Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11451421333333334, "avg_duration_per_task_ms": 146387.56666666668, "avg_output_tokens_per_task": 17546.113333333335, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.30728538666666666, "avg_duration_per_task_ms": 307577.23333333334, "avg_output_tokens_per_task": 10871.333333333334, "last_run_timestamp": "2026-10-17T11:22:31.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.29158895333333334, "avg_duration_per_task_ms": 171961.5, "avg_output_tokens_per_task": 10696.2, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.04354662039954679, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008277793333333333, "avg_duration_per_task_ms": 330908.58666666667, "avg_output_tokens_per_task": 4160.446666666667, "last_run_timestamp": "2026-10-17T11:02:35.316Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7533331666666667, "accuracy_std_dev": 0.0249444271225503, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01177491, "avg_duration_per_task_ms": 456509.82666666666, "avg_output_tokens_per_task": 5746.43, "last_run_timestamp": "2026-10-17T11:16:27.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.032659863237109024, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12209446666666665, "avg_duration_per_task_ms": 147223.81333333332, "avg_output_tokens_per_task": 19831.306666666667, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.04351718, "avg_duration_per_task_ms": 151205.1, "avg_output_tokens_per_task": 15714.7, "last_run_timestamp": "2026-10-17T11:21:27.566Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.027578619999999998, "avg_duration_per_task_ms": 79703.36, "avg_output_tokens_per_task": 19558.22, "last_run_timestamp": "2026-10-17T11:01:12.710Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7528572857142857, "accuracy_std_dev": 0.029543055172381347, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.35331957714285717, "avg_duration_per_task_ms": 231631.49, "avg_output_tokens_per_task": 10894.82142857143, "last_run_timestamp": "2026-10-17T10:59:10.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7528390961923848, "accuracy_std_dev": 0.021447321416904284, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.028104985971943892, "avg_duration_per_task_ms": 567640.0821643287, "avg_output_tokens_per_task": 22541.43887775551, "last_run_timestamp": "2026-10-17T19:46:06.750Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7526667, "accuracy_std_dev": 0.03133329149020257, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014175738, "avg_duration_per_task_ms": 140877.726, "avg_output_tokens_per_task": 16423.73, "last_run_timestamp": "2026-10-17T11:13:51.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.03021638651133634, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5157013133333334, "avg_duration_per_task_ms": 178548.1511111111, "avg_output_tokens_per_task": 7818.084444444445, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7525252929292929, "accuracy_std_dev": 0.03312506718986897, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.10546999326599327, "avg_duration_per_task_ms": 422614.45791245793, "avg_output_tokens_per_task": 27716.097643097644, "last_run_timestamp": "2026-10-17T11:01:37.513Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.752381, "accuracy_std_dev": 0.012046696749376112, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.16778733428571427, "avg_duration_per_task_ms": 474167.0485714286, "avg_output_tokens_per_task": 9790.025714285714, "last_run_timestamp": "2026-10-03T16:16:10.656Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7522322901785714, "accuracy_std_dev": 0.028704344916087695, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.005231792410714286, "avg_duration_per_task_ms": 267977.26339285716, "avg_output_tokens_per_task": 12520.285714285714, "last_run_timestamp": "2026-10-17T19:42:09.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7522221666666666, "accuracy_std_dev": 0.01511278994115765, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.44058496666666663, "avg_duration_per_task_ms": 458389.25, "avg_output_tokens_per_task": 35681.92333333333, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7521212727272727, "accuracy_std_dev": 0.030855459711638198, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5465765, "avg_duration_per_task_ms": 183245.5, "avg_output_tokens_per_task": 10464.481818181817, "last_run_timestamp": "2026-10-17T11:16:11.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.752121090909091, "accuracy_std_dev": 0.019450635878619636, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.037303981818181814, "avg_duration_per_task_ms": 289240.86, "avg_output_tokens_per_task": 22102.725454545456, "last_run_timestamp": "2026-10-17T19:41:19.572Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7520080321285141, "accuracy_std_dev": 0.02504223543536002, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.044945560240963855, "avg_duration_per_task_ms": 491918.7791164659, "avg_output_tokens_per_task": 23149.75502008032, "last_run_timestamp": "2026-10-17T19:15:16.916Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.752, "accuracy_std_dev": 0.022469515152757562, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019928192, "avg_duration_per_task_ms": 74879.332, "avg_output_tokens_per_task": 4253.16, "last_run_timestamp": "2026-10-10T11:24:18.065Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7519999333333334, "accuracy_std_dev": 0.015625282459171393, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.36818162, "avg_duration_per_task_ms": 160625.57733333332, "avg_output_tokens_per_task": 7185.578666666666, "last_run_timestamp": "2026-10-17T10:59:56.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.7516665, "accuracy_std_dev": 0.0327870655403926, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024614285, "avg_duration_per_task_ms": 136581.2, "avg_output_tokens_per_task": 8213.13, "last_run_timestamp": "2026-10-17T10:30:18.157Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7512820769230769, "accuracy_std_dev": 0.022093110003108842, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.009568580000000002, "avg_duration_per_task_ms": 227674.30153846153, "avg_output_tokens_per_task": 4275.484615384616, "last_run_timestamp": "2026-10-17T11:16:25.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7512120909090909, "accuracy_std_dev": 0.028024962309453366, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5612838363636363, "avg_duration_per_task_ms": 202150.13818181818, "avg_output_tokens_per_task": 9898.827272727272, "last_run_timestamp": "2026-10-17T11:07:54.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7511113333333332, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48754586, "avg_duration_per_task_ms": 141651.29333333333, "avg_output_tokens_per_task": 7913.286666666667, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7511112222222222, "accuracy_std_dev": 0.024343305923476197, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19633602444444442, "avg_duration_per_task_ms": 119418.28666666667, "avg_output_tokens_per_task": 7693.504444444445, "last_run_timestamp": "2026-10-17T10:24:17.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.751111, "accuracy_std_dev": 0.03691848879476096, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07572861111111111, "avg_duration_per_task_ms": 225640.9911111111, "avg_output_tokens_per_task": 9556.191111111111, "last_run_timestamp": "2026-10-17T19:39:27.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7510040843373493, "accuracy_std_dev": 0.03597470626982242, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.01730639357429719, "avg_duration_per_task_ms": 141614.70682730924, "avg_output_tokens_per_task": 4001.2329317269077, "last_run_timestamp": "2026-10-17T11:16:03.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7509090909090909, "accuracy_std_dev": 0.021699979478224694, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.32347144, "avg_duration_per_task_ms": 197653.47454545455, "avg_output_tokens_per_task": 11502.629090909091, "last_run_timestamp": "2026-10-17T11:00:48.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7506668, "accuracy_std_dev": 0.025508095447524096, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.354436884, "avg_duration_per_task_ms": 88439.744, "avg_output_tokens_per_task": 7011.238, "last_run_timestamp": "2026-10-17T10:22:34.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.7505556666666666, "accuracy_std_dev": 0.0348230358702716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.014334536666666668, "avg_duration_per_task_ms": 41852.60666666667, "avg_output_tokens_per_task": 4084.32, "last_run_timestamp": "2026-10-17T11:16:45.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7503703333333334, "accuracy_std_dev": 0.03357129710200534, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16148964666666668, "avg_duration_per_task_ms": 232801.40444444446, "avg_output_tokens_per_task": 9705.164444444445, "last_run_timestamp": "2026-10-17T19:58:33.819Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7500001, "accuracy_std_dev": 0.020275803862979145, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.079513518, "avg_duration_per_task_ms": 279626.384, "avg_output_tokens_per_task": 11402.63, "last_run_timestamp": "2026-10-17T23:15:35.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02619736, "avg_duration_per_task_ms": 832580.72, "avg_output_tokens_per_task": 18187.39, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.75, "accuracy_std_dev": 0.04384306609373936, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.50077502, "avg_duration_per_task_ms": 139265.845, "avg_output_tokens_per_task": 7986.47, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.749697, "accuracy_std_dev": 0.04827223001986659, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1384416090909091, "avg_duration_per_task_ms": 852920.6018181818, "avg_output_tokens_per_task": 32022.93090909091, "last_run_timestamp": "2026-10-17T19:47:55.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7496969090909091, "accuracy_std_dev": 0.03721236699235288, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.031196743636363634, "avg_duration_per_task_ms": 263478.21454545454, "avg_output_tokens_per_task": 17098.916363636363, "last_run_timestamp": "2026-10-17T10:59:43.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7496295555555555, "accuracy_std_dev": 0.03543190381597766, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09387901111111112, "avg_duration_per_task_ms": 235629.06666666668, "avg_output_tokens_per_task": 12019.075555555555, "last_run_timestamp": "2026-10-17T20:23:53.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.749166625, "accuracy_std_dev": 0.02559034513218561, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17263283, "avg_duration_per_task_ms": 326399.945, "avg_output_tokens_per_task": 11094.7075, "last_run_timestamp": "2026-10-17T11:01:52.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7490475714285715, "accuracy_std_dev": 0.026500335856939866, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.34032975285714284, "avg_duration_per_task_ms": 209096.78714285715, "avg_output_tokens_per_task": 11852.83, "last_run_timestamp": "2026-10-17T11:11:39.553Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.748889, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012787326666666666, "avg_duration_per_task_ms": 96341.12, "avg_output_tokens_per_task": 11111.46, "last_run_timestamp": "2026-10-17T10:28:17.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.7488888333333333, "accuracy_std_dev": 0.03166158633853052, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.021630793333333332, "avg_duration_per_task_ms": 180688.53666666665, "avg_output_tokens_per_task": 8425.083333333334, "last_run_timestamp": "2026-10-17T19:19:35.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7488737702702704, "accuracy_std_dev": 0.035681240826082414, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.3075426081081082, "avg_duration_per_task_ms": 149494.7972972973, "avg_output_tokens_per_task": 20582.41554054054, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.748484909090909, "accuracy_std_dev": 0.025280931741512393, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08331156545454546, "avg_duration_per_task_ms": 839586.4345454546, "avg_output_tokens_per_task": 36324.976363636364, "last_run_timestamp": "2026-10-17T19:46:44.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7484848181818182, "accuracy_std_dev": 0.027574050564115497, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5215495545454547, "avg_duration_per_task_ms": 226716.27272727274, "avg_output_tokens_per_task": 10061.14, "last_run_timestamp": "2026-10-17T10:58:52.040Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7483332500000001, "accuracy_std_dev": 0.03362363173703132, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10647082, "avg_duration_per_task_ms": 364506.8875, "avg_output_tokens_per_task": 8022.99, "last_run_timestamp": "2026-10-17T19:40:08.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7482990408163266, "accuracy_std_dev": 0.02555550000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.015089622448979592, "avg_duration_per_task_ms": 65496.448979591834, "avg_output_tokens_per_task": 4181.989795918367, "last_run_timestamp": "2026-10-17T10:59:16.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7481482222222222, "accuracy_std_dev": 0.025293761601442692, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.34738337555555554, "avg_duration_per_task_ms": 200384.05333333334, "avg_output_tokens_per_task": 11864.717777777778, "last_run_timestamp": "2026-10-17T11:13:53.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7481481111111111, "accuracy_std_dev": 0.038332810160036944, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11396319111111111, "avg_duration_per_task_ms": 269132.9111111111, "avg_output_tokens_per_task": 9467.16888888889, "last_run_timestamp": "2026-10-17T19:41:15.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7481398080357142, "accuracy_std_dev": 0.020714088201028814, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.11667296428571428, "avg_duration_per_task_ms": 119172.515625, "avg_output_tokens_per_task": 10392.310267857143, "last_run_timestamp": "2026-10-17T19:14:08.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7479999, "accuracy_std_dev": 0.04171853092200154, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.041866538, "avg_duration_per_task_ms": 196873.67, "avg_output_tokens_per_task": 19363.92, "last_run_timestamp": "2026-10-17T11:00:50.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.02878187658589265, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023901795, "avg_duration_per_task_ms": 159481.33333333334, "avg_output_tokens_per_task": 7971.18, "last_run_timestamp": "2026-10-17T19:25:51.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7475, "accuracy_std_dev": 0.016811884025890747, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01307641, "avg_duration_per_task_ms": 361470.8875, "avg_output_tokens_per_task": 16341.3725, "last_run_timestamp": "2026-10-17T11:13:02.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7474074444444444, "accuracy_std_dev": 0.030216386511336353, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07081323111111111, "avg_duration_per_task_ms": 255490.43777777778, "avg_output_tokens_per_task": 9828.68888888889, "last_run_timestamp": "2026-10-17T19:51:09.246Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7474073333333333, "accuracy_std_dev": 0.013496814644780307, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013488784444444445, "avg_duration_per_task_ms": 102984.29111111112, "avg_output_tokens_per_task": 10999.817777777778, "last_run_timestamp": "2026-10-17T11:00:48.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7473841614349777, "accuracy_std_dev": 0.04596493608054828, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.07238068609865472, "avg_duration_per_task_ms": 340205.40134529147, "avg_output_tokens_per_task": 12677.688340807175, "last_run_timestamp": "2026-10-17T20:02:16.814Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7472221666666666, "accuracy_std_dev": 0.011613686586906374, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17439429333333334, "avg_duration_per_task_ms": 161420.59666666668, "avg_output_tokens_per_task": 12499.9, "last_run_timestamp": "2026-10-10T11:19:48.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7472220833333334, "accuracy_std_dev": 0.022187493002659323, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.013829399999999999, "avg_duration_per_task_ms": 128044.86833333333, "avg_output_tokens_per_task": 15864.645, "last_run_timestamp": "2026-10-17T10:46:05.123Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7471783002257337, "accuracy_std_dev": 0.040296469031969095, "run_count": 9, "total_questions": 443, "avg_cost_per_task_usd": 0.11695101128668173, "avg_duration_per_task_ms": 466244.8329571106, "avg_output_tokens_per_task": 21735.31828442438, "last_run_timestamp": "2026-10-03T08:18:33.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.747037111111111, "accuracy_std_dev": 0.03619039918555579, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07854862444444444, "avg_duration_per_task_ms": 352663.33777777775, "avg_output_tokens_per_task": 9208.286666666667, "last_run_timestamp": "2026-10-17T19:38:35.311Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7468716348122866, "accuracy_std_dev": 0.026415158764534374, "run_count": 6, "total_questions": 293, "avg_cost_per_task_usd": 0.1360147542662116, "avg_duration_per_task_ms": 219042.409556314, "avg_output_tokens_per_task": 11992.477815699658, "last_run_timestamp": "2026-10-17T19:36:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05210678, "avg_duration_per_task_ms": 128609.46, "avg_output_tokens_per_task": 18103.84, "last_run_timestamp": "2026-10-17T11:13:14.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.02687417182328601, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07283523750000001, "avg_duration_per_task_ms": 314827.96, "avg_output_tokens_per_task": 11539.3075, "last_run_timestamp": "2026-10-17T23:23:01.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.023726809342082992, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21412848666666662, "avg_duration_per_task_ms": 142371.99333333335, "avg_output_tokens_per_task": 22756.126666666667, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18399340666666666, "avg_duration_per_task_ms": 116870.40666666666, "avg_output_tokens_per_task": 5117.2266666666665, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.746666625, "accuracy_std_dev": 0.03179809568172874, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4209529125, "avg_duration_per_task_ms": 657671.5425, "avg_output_tokens_per_task": 9874.79, "last_run_timestamp": "2026-10-17T10:59:15.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.746666625, "accuracy_std_dev": 0.025165982331996805, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0084872125, "avg_duration_per_task_ms": 222581.33, "avg_output_tokens_per_task": 5098.45, "last_run_timestamp": "2026-10-17T11:55:03.480Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7466666000000001, "accuracy_std_dev": 0.012649321461643697, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.35012448, "avg_duration_per_task_ms": 129259.12, "avg_output_tokens_per_task": 7227.552, "last_run_timestamp": "2026-10-17T10:38:33.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7466666000000001, "accuracy_std_dev": 0.04226904304618216, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.011620872, "avg_duration_per_task_ms": 319057.722, "avg_output_tokens_per_task": 16204.506, "last_run_timestamp": "2026-10-17T11:16:46.243Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.7466665555555556, "accuracy_std_dev": 0.019116420452416828, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013255473333333333, "avg_duration_per_task_ms": 72684.63333333333, "avg_output_tokens_per_task": 11005.913333333334, "last_run_timestamp": "2026-10-17T11:00:48.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7464314105793451, "accuracy_std_dev": 0.030785849417538806, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.06752257934508815, "avg_duration_per_task_ms": 443836.2191435768, "avg_output_tokens_per_task": 10462.246851385391, "last_run_timestamp": "2026-10-10T18:57:24.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.7461111666666667, "accuracy_std_dev": 0.018147215208002467, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.016089411666666668, "avg_duration_per_task_ms": 175782.82666666666, "avg_output_tokens_per_task": 4282.083333333333, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7459256666666667, "accuracy_std_dev": 0.02542365100103096, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08970129333333333, "avg_duration_per_task_ms": 241877.4, "avg_output_tokens_per_task": 10154.835555555555, "last_run_timestamp": "2026-10-17T20:11:09.874Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.745833375, "accuracy_std_dev": 0.03357197181778836, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1318732225, "avg_duration_per_task_ms": 359914.9725, "avg_output_tokens_per_task": 11863.2075, "last_run_timestamp": "2026-10-17T19:38:35.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7458124924623115, "accuracy_std_dev": 0.02659090084943297, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.0608610351758794, "avg_duration_per_task_ms": 201989.4798994975, "avg_output_tokens_per_task": 32682.85175879397, "last_run_timestamp": "2026-10-17T10:49:21.111Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7456411538461538, "accuracy_std_dev": 0.027280880543103812, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.0642081323076923, "avg_duration_per_task_ms": 143388.6953846154, "avg_output_tokens_per_task": 9780.298461538461, "last_run_timestamp": "2026-10-17T19:39:04.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7454957162162162, "accuracy_std_dev": 0.015757653442057918, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.12553497972972974, "avg_duration_per_task_ms": 598677.7297297297, "avg_output_tokens_per_task": 24585.641891891893, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7453763225806453, "accuracy_std_dev": 0.03210277114898852, "run_count": 31, "total_questions": 1550, "avg_cost_per_task_usd": 0.3755844303225807, "avg_duration_per_task_ms": 200838.47032258066, "avg_output_tokens_per_task": 11986.134838709677, "last_run_timestamp": "2026-10-17T11:25:08.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7452106666666666, "accuracy_std_dev": 0.020011549577426443, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.047182301724137926, "avg_duration_per_task_ms": 787118.643678161, "avg_output_tokens_per_task": 14354.25, "last_run_timestamp": "2026-10-17T19:16:29.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7450001666666666, "accuracy_std_dev": 0.024324256298440514, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017958773333333337, "avg_duration_per_task_ms": 482242.6666666667, "avg_output_tokens_per_task": 25086.3, "last_run_timestamp": "2026-10-17T11:07:54.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.745, "accuracy_std_dev": 0.02020721818435187, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.010433062500000001, "avg_duration_per_task_ms": 212069.0525, "avg_output_tokens_per_task": 14202.0475, "last_run_timestamp": "2026-10-17T19:19:49.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.745, "accuracy_std_dev": 0.015184019378939179, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.044957634999999996, "avg_duration_per_task_ms": 186177.835, "avg_output_tokens_per_task": 30974.63, "last_run_timestamp": "2026-10-17T11:15:59.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.745, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15284815, "avg_duration_per_task_ms": 494527.49, "avg_output_tokens_per_task": 28246.935, "last_run_timestamp": "2026-10-17T19:47:21.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.745, "accuracy_std_dev": 0.015122928387055215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03447282333333334, "avg_duration_per_task_ms": 421378.29333333333, "avg_output_tokens_per_task": 9247.876666666667, "last_run_timestamp": "2026-10-09T02:05:38.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.744993393858478, "accuracy_std_dev": 0.01640917934761584, "run_count": 15, "total_questions": 749, "avg_cost_per_task_usd": 0.04752114152202937, "avg_duration_per_task_ms": 285856.17356475303, "avg_output_tokens_per_task": 27467.751668891855, "last_run_timestamp": "2026-10-17T19:35:48.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7444864177693761, "accuracy_std_dev": 0.0319646756633704, "run_count": 11, "total_questions": 529, "avg_cost_per_task_usd": 0.02029702079395085, "avg_duration_per_task_ms": 91063.5595463138, "avg_output_tokens_per_task": 6679.818525519849, "last_run_timestamp": "2026-10-17T18:34:28.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2319787333333332, "avg_duration_per_task_ms": 98531.56, "avg_output_tokens_per_task": 5524.42, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.010657241210661543, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012899976666666667, "avg_duration_per_task_ms": 76869.22333333333, "avg_output_tokens_per_task": 10868.43, "last_run_timestamp": "2026-10-17T10:31:56.734Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.74416675, "accuracy_std_dev": 0.03012731088543251, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.064155875, "avg_duration_per_task_ms": 1722921.32, "avg_output_tokens_per_task": 8915.945, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7441076363636364, "accuracy_std_dev": 0.008720982312790204, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.2539274696969697, "avg_duration_per_task_ms": 431832.7626262626, "avg_output_tokens_per_task": 12531.59090909091, "last_run_timestamp": "2026-10-17T10:59:55.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7440801639344262, "accuracy_std_dev": 0.02729307539006798, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.011795925318761384, "avg_duration_per_task_ms": 451361.3533697632, "avg_output_tokens_per_task": 14179.424408014573, "last_run_timestamp": "2026-10-17T10:31:56.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.743889, "accuracy_std_dev": 0.03872595709813596, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02433945333333333, "avg_duration_per_task_ms": 401395.9033333333, "avg_output_tokens_per_task": 15168.051666666666, "last_run_timestamp": "2026-10-17T11:15:55.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7438888333333333, "accuracy_std_dev": 0.03327303342256143, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.027845846666666667, "avg_duration_per_task_ms": 428793.8, "avg_output_tokens_per_task": 14516.615, "last_run_timestamp": "2026-10-17T13:55:17.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7438479642058166, "accuracy_std_dev": 0.03102502627609039, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.06547314988814318, "avg_duration_per_task_ms": 990556.5682326622, "avg_output_tokens_per_task": 71441.88143176734, "last_run_timestamp": "2026-10-17T19:52:46.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7438097142857142, "accuracy_std_dev": 0.03525096933505834, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.015975259999999998, "avg_duration_per_task_ms": 149636.74857142856, "avg_output_tokens_per_task": 15244.568571428572, "last_run_timestamp": "2026-10-17T10:31:13.895Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7437710606060606, "accuracy_std_dev": 0.022759018292755954, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.024460747474747482, "avg_duration_per_task_ms": 280677.9696969697, "avg_output_tokens_per_task": 18749.616161616163, "last_run_timestamp": "2026-10-17T11:00:48.266Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.019907322310033455, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02349562666666667, "avg_duration_per_task_ms": 145712.99333333335, "avg_output_tokens_per_task": 16978.89, "last_run_timestamp": "2026-10-17T10:59:45.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.01452975873638307, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1182949875, "avg_duration_per_task_ms": 350271.9625, "avg_output_tokens_per_task": 9768.27, "last_run_timestamp": "2026-10-17T20:28:21.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7433035357142856, "accuracy_std_dev": 0.02587302115436699, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.03511485267857143, "avg_duration_per_task_ms": 607727.7254464285, "avg_output_tokens_per_task": 15955.859375, "last_run_timestamp": "2026-10-17T19:37:14.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7431110666666667, "accuracy_std_dev": 0.017866552793666977, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.3618751893333333, "avg_duration_per_task_ms": 88230.65866666667, "avg_output_tokens_per_task": 7156.333333333333, "last_run_timestamp": "2026-10-17T10:59:45.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.742962888888889, "accuracy_std_dev": 0.028993307965208676, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3506727244444445, "avg_duration_per_task_ms": 221762.51111111112, "avg_output_tokens_per_task": 10894.351111111111, "last_run_timestamp": "2026-10-17T10:58:41.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7428571904761905, "accuracy_std_dev": 0.020426289140687226, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.4526462142857143, "avg_duration_per_task_ms": 607353.6895238095, "avg_output_tokens_per_task": 9902.753333333334, "last_run_timestamp": "2026-10-18T01:21:52.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7426667, "accuracy_std_dev": 0.031860777699390844, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.107968072, "avg_duration_per_task_ms": 317982.848, "avg_output_tokens_per_task": 18593.702, "last_run_timestamp": "2026-10-17T11:15:57.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.006285707880227596, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.38184122000000004, "avg_duration_per_task_ms": 149071.25333333333, "avg_output_tokens_per_task": 12794.173333333334, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0028365266666666666, "avg_duration_per_task_ms": 68565.14666666667, "avg_output_tokens_per_task": 11675.026666666667, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7422222222222222, "accuracy_std_dev": 0.05057690391819785, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09490620666666666, "avg_duration_per_task_ms": 230369.54888888888, "avg_output_tokens_per_task": 11349.884444444444, "last_run_timestamp": "2026-10-17T20:40:39.123Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.03541660697289089, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.026421163333333334, "avg_duration_per_task_ms": 160993.86333333334, "avg_output_tokens_per_task": 16769.46, "last_run_timestamp": "2026-10-17T10:59:46.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.016178166968857207, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05587016000000001, "avg_duration_per_task_ms": 223603.79666666666, "avg_output_tokens_per_task": 9185.596666666666, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.742222, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.044708606666666664, "avg_duration_per_task_ms": 149318.53333333333, "avg_output_tokens_per_task": 38342.77333333333, "last_run_timestamp": "2026-10-10T11:04:02.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7421122716049383, "accuracy_std_dev": 0.04949233809146621, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.07618280658436215, "avg_duration_per_task_ms": 605963.6255144033, "avg_output_tokens_per_task": 7686.909465020576, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.742083375, "accuracy_std_dev": 0.02948325972877448, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0861245475, "avg_duration_per_task_ms": 179001.195, "avg_output_tokens_per_task": 10461.4425, "last_run_timestamp": "2026-10-17T19:46:06.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7418181818181818, "accuracy_std_dev": 0.108770519591242, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1487734618181818, "avg_duration_per_task_ms": 616954.0145454545, "avg_output_tokens_per_task": 27127.465454545454, "last_run_timestamp": "2026-10-17T11:01:24.247Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7418154508928572, "accuracy_std_dev": 0.028897426433319922, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.09664293303571428, "avg_duration_per_task_ms": 357228.20089285716, "avg_output_tokens_per_task": 11297.408482142857, "last_run_timestamp": "2026-10-17T19:51:54.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7417582051282051, "accuracy_std_dev": 0.022765247487641204, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.1424760476190476, "avg_duration_per_task_ms": 619826.9175824176, "avg_output_tokens_per_task": 26765.48534798535, "last_run_timestamp": "2026-10-17T19:26:00.646Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7416668333333333, "accuracy_std_dev": 0.0247767924573021, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04222387666666667, "avg_duration_per_task_ms": 129284.56, "avg_output_tokens_per_task": 26313.053333333333, "last_run_timestamp": "2026-10-17T10:27:15.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7416668333333333, "accuracy_std_dev": 0.03963955422687743, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.00980467, "avg_duration_per_task_ms": 415763.42833333334, "avg_output_tokens_per_task": 14671.45, "last_run_timestamp": "2026-10-17T11:26:35.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.741666625, "accuracy_std_dev": 0.037675458046112394, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.3525293925, "avg_duration_per_task_ms": 391694.48, "avg_output_tokens_per_task": 9580.7525, "last_run_timestamp": "2026-10-17T11:00:48.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.7414815555555555, "accuracy_std_dev": 0.03675450160520359, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04532596444444444, "avg_duration_per_task_ms": 571751.6577777778, "avg_output_tokens_per_task": 9173.444444444445, "last_run_timestamp": "2026-10-17T20:41:08.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7414814444444444, "accuracy_std_dev": 0.03043362969228142, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5050336355555556, "avg_duration_per_task_ms": 191771.47333333333, "avg_output_tokens_per_task": 9950.68888888889, "last_run_timestamp": "2026-10-17T11:12:58.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7414813333333333, "accuracy_std_dev": 0.021029672681977536, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06887691555555556, "avg_duration_per_task_ms": 310202.34444444446, "avg_output_tokens_per_task": 12962.453333333333, "last_run_timestamp": "2026-10-17T19:46:38.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7413334, "accuracy_std_dev": 0.02578533261061415, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.064516224, "avg_duration_per_task_ms": 309719.048, "avg_output_tokens_per_task": 8304.048, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7411113333333332, "accuracy_std_dev": 0.00415742679497253, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08487537333333334, "avg_duration_per_task_ms": 315752.68666666665, "avg_output_tokens_per_task": 9327.993333333334, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.740963843373494, "accuracy_std_dev": 0.0367474030332757, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.079212578313253, "avg_duration_per_task_ms": 799563.951807229, "avg_output_tokens_per_task": 14536.901606425703, "last_run_timestamp": "2026-10-17T21:16:39.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7405555, "accuracy_std_dev": 0.028474594482040785, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.021673546666666665, "avg_duration_per_task_ms": 395217.175, "avg_output_tokens_per_task": 12192.15, "last_run_timestamp": "2026-10-17T11:29:07.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7403703333333334, "accuracy_std_dev": 0.030325141571529955, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3409674066666667, "avg_duration_per_task_ms": 128710.22, "avg_output_tokens_per_task": 5246.528888888889, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7403200757575757, "accuracy_std_dev": 0.01686858005610948, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.018282113636363638, "avg_duration_per_task_ms": 87170.2398989899, "avg_output_tokens_per_task": 4071.1969696969695, "last_run_timestamp": "2026-10-17T11:12:55.113Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.7400003333333333, "accuracy_std_dev": 0.018856180831641253, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.019302796666666667, "avg_duration_per_task_ms": 499555.5366666667, "avg_output_tokens_per_task": 14674.223333333333, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400002, "accuracy_std_dev": 0.01264900523203309, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034869184, "avg_duration_per_task_ms": 129959.044, "avg_output_tokens_per_task": 16349.368, "last_run_timestamp": "2026-10-03T08:21:01.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7400001428571428, "accuracy_std_dev": 0.031471771176398565, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07991938000000001, "avg_duration_per_task_ms": 532201.7285714286, "avg_output_tokens_per_task": 12014.771428571428, "last_run_timestamp": "2026-10-17T23:14:09.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08086026, "avg_duration_per_task_ms": 522771.74, "avg_output_tokens_per_task": 13545.48, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.049757800000000005, "avg_duration_per_task_ms": 211927.58, "avg_output_tokens_per_task": 8527.66, "last_run_timestamp": "2026-07-25T08:07:26.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.74, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.15468345, "avg_duration_per_task_ms": 99212.54, "avg_output_tokens_per_task": 8743.23, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.74, "accuracy_std_dev": 0.044555500139339316, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.0935073933333332, "avg_duration_per_task_ms": 295552.94, "avg_output_tokens_per_task": 22860.126666666667, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.74, "accuracy_std_dev": 0.01671431284857383, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.2243457, "avg_duration_per_task_ms": 261652.61428571428, "avg_output_tokens_per_task": 11882.217142857144, "last_run_timestamp": "2026-10-17T11:03:43.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.74, "accuracy_std_dev": 0.03858593583807446, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.42733946500000003, "avg_duration_per_task_ms": 38766.985, "avg_output_tokens_per_task": 9216.565, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03319066, "avg_duration_per_task_ms": 77162.14, "avg_output_tokens_per_task": 4453.24, "last_run_timestamp": "2026-10-04T03:00:16.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7399998571428572, "accuracy_std_dev": 0.023094052007442304, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01886730285714286, "avg_duration_per_task_ms": 206247.68, "avg_output_tokens_per_task": 4550.928571428572, "last_run_timestamp": "2026-10-10T11:50:34.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7394871538461538, "accuracy_std_dev": 0.027606692440688974, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.056419910769230766, "avg_duration_per_task_ms": 240446.07846153845, "avg_output_tokens_per_task": 9604.89076923077, "last_run_timestamp": "2026-10-17T19:41:59.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7394766996587031, "accuracy_std_dev": 0.028748585800363012, "run_count": 12, "total_questions": 586, "avg_cost_per_task_usd": 0.06203791467576791, "avg_duration_per_task_ms": 129086.0494880546, "avg_output_tokens_per_task": 8186.447098976109, "last_run_timestamp": "2026-10-17T19:18:24.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7393334, "accuracy_std_dev": 0.049121084860169785, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009324213999999999, "avg_duration_per_task_ms": 313360.034, "avg_output_tokens_per_task": 15426.052, "last_run_timestamp": "2026-10-17T11:16:23.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7392550315186246, "accuracy_std_dev": 0.025013223548469583, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.23634143553008594, "avg_duration_per_task_ms": 371772.38968481374, "avg_output_tokens_per_task": 10117.719197707736, "last_run_timestamp": "2026-10-17T11:09:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7390475714285715, "accuracy_std_dev": 0.03175881615667484, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.057812691428571435, "avg_duration_per_task_ms": 661457.1457142857, "avg_output_tokens_per_task": 14926.3, "last_run_timestamp": "2026-10-10T19:35:42.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7388889166666667, "accuracy_std_dev": 0.0286530189318634, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023847688333333335, "avg_duration_per_task_ms": 88794.73166666667, "avg_output_tokens_per_task": 7901.106666666667, "last_run_timestamp": "2026-10-17T19:36:03.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7387772808022923, "accuracy_std_dev": 0.02413990130003665, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.05359102578796561, "avg_duration_per_task_ms": 822214.4985673352, "avg_output_tokens_per_task": 14206.361031518625, "last_run_timestamp": "2026-10-17T19:36:08.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7386668, "accuracy_std_dev": 0.03166503013357166, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009092168000000001, "avg_duration_per_task_ms": 101935.708, "avg_output_tokens_per_task": 5027.116, "last_run_timestamp": "2026-10-17T11:17:02.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.017992647486688756, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3103866155555556, "avg_duration_per_task_ms": 191198.25333333333, "avg_output_tokens_per_task": 10879.702222222222, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.738333375, "accuracy_std_dev": 0.015898903703852527, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07332558500000001, "avg_duration_per_task_ms": 262220.325, "avg_output_tokens_per_task": 13184.325, "last_run_timestamp": "2026-10-17T19:21:39.847Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7383231627502177, "accuracy_std_dev": 0.027430622924368746, "run_count": 23, "total_questions": 1149, "avg_cost_per_task_usd": 0.12736605570060922, "avg_duration_per_task_ms": 169094.96257615316, "avg_output_tokens_per_task": 9298.362924281984, "last_run_timestamp": "2026-10-17T19:52:56.176Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7382865983935742, "accuracy_std_dev": 0.027461257286584682, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.07498479919678715, "avg_duration_per_task_ms": 91080.29718875502, "avg_output_tokens_per_task": 4367.955823293173, "last_run_timestamp": "2026-09-26T08:11:26.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7382021797752809, "accuracy_std_dev": 0.021880590311850887, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.020498339325842697, "avg_duration_per_task_ms": 307841.11235955055, "avg_output_tokens_per_task": 10994.101123595505, "last_run_timestamp": "2026-10-17T11:09:59.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.7378469999999999, "accuracy_std_dev": 0.0017360000000000153, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.21588738541666666, "avg_duration_per_task_ms": 378388.65625, "avg_output_tokens_per_task": 25153.166666666668, "last_run_timestamp": "2026-10-17T19:37:05.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7375, "accuracy_std_dev": 0.04795095874432546, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17019316, "avg_duration_per_task_ms": 283667.8625, "avg_output_tokens_per_task": 9793.12, "last_run_timestamp": "2026-10-17T11:16:12.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.737499875, "accuracy_std_dev": 0.018837726259009493, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0148281175, "avg_duration_per_task_ms": 76477.91, "avg_output_tokens_per_task": 4476.9775, "last_run_timestamp": "2026-10-17T10:59:31.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7372220833333334, "accuracy_std_dev": 0.023484995867781692, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.007983868333333335, "avg_duration_per_task_ms": 398396.11666666664, "avg_output_tokens_per_task": 14403.198333333334, "last_run_timestamp": "2026-10-17T12:09:02.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7370185075376885, "accuracy_std_dev": 0.05274392638908769, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.2985063944723619, "avg_duration_per_task_ms": 383061.77889447234, "avg_output_tokens_per_task": 11160.087939698493, "last_run_timestamp": "2026-10-17T11:17:31.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7369696818181818, "accuracy_std_dev": 0.024677843497183632, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.36901798272727276, "avg_duration_per_task_ms": 230204.26454545456, "avg_output_tokens_per_task": 10920.546363636364, "last_run_timestamp": "2026-10-17T11:09:53.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2879983000000002, "avg_duration_per_task_ms": 310547.73, "avg_output_tokens_per_task": 22913.4, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7366668, "accuracy_std_dev": 0.022900761755015906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.063241744, "avg_duration_per_task_ms": 1530537.112, "avg_output_tokens_per_task": 25222.716, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.736666625, "accuracy_std_dev": 0.03464100412422215, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09860756999999999, "avg_duration_per_task_ms": 640483.4725, "avg_output_tokens_per_task": 9709.82, "last_run_timestamp": "2026-10-17T21:09:49.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7362964444444444, "accuracy_std_dev": 0.017808576717170822, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.38587963333333336, "avg_duration_per_task_ms": 178574.04, "avg_output_tokens_per_task": 10909.295555555556, "last_run_timestamp": "2026-10-10T11:25:32.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7362962222222222, "accuracy_std_dev": 0.05087965035645451, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.047357815555555544, "avg_duration_per_task_ms": 286314.82666666666, "avg_output_tokens_per_task": 14790.351111111111, "last_run_timestamp": "2026-10-17T11:21:16.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7361904285714286, "accuracy_std_dev": 0.022213041869632365, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.35529426857142854, "avg_duration_per_task_ms": 193600.25714285715, "avg_output_tokens_per_task": 8821.457142857143, "last_run_timestamp": "2026-10-17T10:59:54.343Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7361110833333333, "accuracy_std_dev": 0.04367039035673625, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.082194435, "avg_duration_per_task_ms": 179037.31666666668, "avg_output_tokens_per_task": 8256.305, "last_run_timestamp": "2026-10-17T19:21:01.874Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7360831871227365, "accuracy_std_dev": 0.029580267931342347, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.07848097384305834, "avg_duration_per_task_ms": 216753.75050301812, "avg_output_tokens_per_task": 8419.167002012073, "last_run_timestamp": "2026-10-17T19:14:56.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7357142857142858, "accuracy_std_dev": 0.03689182323153652, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07578944857142858, "avg_duration_per_task_ms": 210085.71714285715, "avg_output_tokens_per_task": 11277.097142857143, "last_run_timestamp": "2026-10-10T18:54:49.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.036649745254533794, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.022963626666666664, "avg_duration_per_task_ms": 113662.22333333333, "avg_output_tokens_per_task": 7480.291666666667, "last_run_timestamp": "2026-10-17T19:17:31.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7354165, "accuracy_std_dev": 0.0429125354005097, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.006874592499999999, "avg_duration_per_task_ms": 421319.53, "avg_output_tokens_per_task": 14336.97, "last_run_timestamp": "2026-10-10T15:31:23.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7351189375, "accuracy_std_dev": 0.09797950654998103, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07525125669642858, "avg_duration_per_task_ms": 194209.64732142858, "avg_output_tokens_per_task": 24190.323660714286, "last_run_timestamp": "2026-10-17T19:25:40.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7350427912087912, "accuracy_std_dev": 0.06405752324575284, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.1534336978021978, "avg_duration_per_task_ms": 1367021.8919413919, "avg_output_tokens_per_task": 27730.919413919415, "last_run_timestamp": "2026-10-17T19:42:10.212Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.735000125, "accuracy_std_dev": 0.028819160776458658, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0161670325, "avg_duration_per_task_ms": 638661.92, "avg_output_tokens_per_task": 18640.1425, "last_run_timestamp": "2026-10-18T01:24:51.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.735000125, "accuracy_std_dev": 0.026405000054144565, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.29933707499999995, "avg_duration_per_task_ms": 500487.98, "avg_output_tokens_per_task": 9623.9275, "last_run_timestamp": "2026-10-17T21:38:40.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.735, "accuracy_std_dev": 0.011666999999999983, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.024598469999999997, "avg_duration_per_task_ms": 93010.71, "avg_output_tokens_per_task": 31378.94, "last_run_timestamp": "2026-10-17T19:09:13.040Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7349997500000001, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12295462999999998, "avg_duration_per_task_ms": 89986.35, "avg_output_tokens_per_task": 23350.75, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7347416579476861, "accuracy_std_dev": 0.025978725188315156, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.35989230985915494, "avg_duration_per_task_ms": 406164.88732394367, "avg_output_tokens_per_task": 9488.581488933602, "last_run_timestamp": "2026-10-17T10:59:45.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.7344445, "accuracy_std_dev": 0.028393141670422214, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04939556333333334, "avg_duration_per_task_ms": 469523.68666666665, "avg_output_tokens_per_task": 9528.593333333334, "last_run_timestamp": "2026-10-17T20:03:15.965Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7344348103448277, "accuracy_std_dev": 0.030951629540778514, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.1064283620689655, "avg_duration_per_task_ms": 100216.88362068965, "avg_output_tokens_per_task": 7768.466954022989, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7342658006993007, "accuracy_std_dev": 0.01683879230048547, "run_count": 6, "total_questions": 286, "avg_cost_per_task_usd": 0.14595530419580421, "avg_duration_per_task_ms": 709836.5559440559, "avg_output_tokens_per_task": 29021.49300699301, "last_run_timestamp": "2026-10-17T19:00:48.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.73416675, "accuracy_std_dev": 0.027975611900144373, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05059770500000001, "avg_duration_per_task_ms": 367749.4825, "avg_output_tokens_per_task": 10213.9225, "last_run_timestamp": "2026-10-17T20:14:14.955Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.734166625, "accuracy_std_dev": 0.039077836323603884, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12202924000000001, "avg_duration_per_task_ms": 675600.895, "avg_output_tokens_per_task": 33746.605, "last_run_timestamp": "2026-10-17T10:59:46.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7341665, "accuracy_std_dev": 0.021650731321366502, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.065515015, "avg_duration_per_task_ms": 151485.83, "avg_output_tokens_per_task": 7332.795, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.73413336, "accuracy_std_dev": 0.03348342146421718, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.3556991112, "avg_duration_per_task_ms": 207103.4464, "avg_output_tokens_per_task": 10323.1728, "last_run_timestamp": "2026-10-17T11:02:34.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7338888333333333, "accuracy_std_dev": 0.02530674652746884, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07665837666666667, "avg_duration_per_task_ms": 177081.09, "avg_output_tokens_per_task": 8668.4, "last_run_timestamp": "2026-10-17T19:13:29.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7338531336302895, "accuracy_std_dev": 0.03071493333547167, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.06568974832962138, "avg_duration_per_task_ms": 161273.1714922049, "avg_output_tokens_per_task": 11527.013363028953, "last_run_timestamp": "2026-10-17T19:36:06.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7333336666666668, "accuracy_std_dev": 0.03771236166328251, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.44470873333333333, "avg_duration_per_task_ms": 130557.96, "avg_output_tokens_per_task": 20380.893333333333, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08394800999999999, "avg_duration_per_task_ms": 134148.76, "avg_output_tokens_per_task": 8723.94, "last_run_timestamp": "2026-10-17T19:55:39.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05760637824832321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04898614, "avg_duration_per_task_ms": 106829.63333333333, "avg_output_tokens_per_task": 9897.326666666666, "last_run_timestamp": "2026-10-10T11:51:48.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.73333325, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17039939499999998, "avg_duration_per_task_ms": 114141.445, "avg_output_tokens_per_task": 5083.04, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7331495477941177, "accuracy_std_dev": 0.04512078785249381, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.02227737316176471, "avg_duration_per_task_ms": 99731.28860294117, "avg_output_tokens_per_task": 7413.3125, "last_run_timestamp": "2026-10-17T19:12:23.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02503509736024079, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06745117142857142, "avg_duration_per_task_ms": 76562.92857142857, "avg_output_tokens_per_task": 8496.594285714285, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.732857, "accuracy_std_dev": 0.02930024262210617, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.015632725714285714, "avg_duration_per_task_ms": 286314.63714285713, "avg_output_tokens_per_task": 22301.08, "last_run_timestamp": "2026-10-17T11:22:26.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7325, "accuracy_std_dev": 0.04596937990662913, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012367795, "avg_duration_per_task_ms": 953069.875, "avg_output_tokens_per_task": 22737, "last_run_timestamp": "2026-10-17T10:32:07.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7322221666666667, "accuracy_std_dev": 0.02705733510169758, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04443540666666667, "avg_duration_per_task_ms": 142368.75, "avg_output_tokens_per_task": 22198.636666666665, "last_run_timestamp": "2026-10-17T11:01:48.208Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7322221666666667, "accuracy_std_dev": 0.027866575847639113, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03493169, "avg_duration_per_task_ms": 340890.4033333333, "avg_output_tokens_per_task": 8339.266666666666, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73166675, "accuracy_std_dev": 0.015184321229067165, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.29879598, "avg_duration_per_task_ms": 177541.565, "avg_output_tokens_per_task": 10870.785, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.731515090909091, "accuracy_std_dev": 0.03470991263301791, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3032066236363637, "avg_duration_per_task_ms": 332655.4072727273, "avg_output_tokens_per_task": 9387.027272727273, "last_run_timestamp": "2026-10-17T10:59:57.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7313334, "accuracy_std_dev": 0.012666852633547132, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.243754672, "avg_duration_per_task_ms": 268035.924, "avg_output_tokens_per_task": 10219.814, "last_run_timestamp": "2026-10-17T11:01:22.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7311113333333332, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10536965333333334, "avg_duration_per_task_ms": 88766.42666666667, "avg_output_tokens_per_task": 4017.0533333333333, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7311112222222222, "accuracy_std_dev": 0.022443432322063868, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05785997111111111, "avg_duration_per_task_ms": 325880.12666666665, "avg_output_tokens_per_task": 9968.124444444444, "last_run_timestamp": "2026-10-17T20:02:15.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.026010406276463026, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03343346333333334, "avg_duration_per_task_ms": 98916.58333333333, "avg_output_tokens_per_task": 23693.51, "last_run_timestamp": "2026-10-17T11:03:07.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.02096437307274735, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04746591, "avg_duration_per_task_ms": 142455.40666666668, "avg_output_tokens_per_task": 28668.966666666667, "last_run_timestamp": "2026-10-17T10:58:45.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.022662199996175744, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3988358066666667, "avg_duration_per_task_ms": 172929.82666666666, "avg_output_tokens_per_task": 27148.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7305553333333333, "accuracy_std_dev": 0.020942233896018736, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04416758666666667, "avg_duration_per_task_ms": 171467.14, "avg_output_tokens_per_task": 30432.296666666665, "last_run_timestamp": "2026-10-17T11:17:21.507Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7299999166666666, "accuracy_std_dev": 0.034047910145015584, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09264119166666665, "avg_duration_per_task_ms": 141837.91666666666, "avg_output_tokens_per_task": 9263.338333333333, "last_run_timestamp": "2026-10-17T19:23:11.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7296969090909091, "accuracy_std_dev": 0.03415505130871314, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.33621692363636363, "avg_duration_per_task_ms": 207110.29636363636, "avg_output_tokens_per_task": 10400.149090909092, "last_run_timestamp": "2026-10-17T10:58:55.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.729166625, "accuracy_std_dev": 0.026020732920776363, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12688592750000002, "avg_duration_per_task_ms": 249207.73, "avg_output_tokens_per_task": 10044.14, "last_run_timestamp": "2026-10-17T21:29:38.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.728889, "accuracy_std_dev": 0.028631520229751458, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05385624666666666, "avg_duration_per_task_ms": 289115.72444444447, "avg_output_tokens_per_task": 11182.26, "last_run_timestamp": "2026-10-17T19:41:41.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7284644516853932, "accuracy_std_dev": 0.03760340361105365, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.09336272584269663, "avg_duration_per_task_ms": 298064.6898876405, "avg_output_tokens_per_task": 14598.31011235955, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7284264213197968, "accuracy_std_dev": 0.023635771369896082, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.051129340101522834, "avg_duration_per_task_ms": 77511.28934010152, "avg_output_tokens_per_task": 19306.862944162436, "last_run_timestamp": "2026-10-17T10:21:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7283333333333334, "accuracy_std_dev": 0.027369352961945523, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08085183000000001, "avg_duration_per_task_ms": 227378.24666666667, "avg_output_tokens_per_task": 8744.926666666666, "last_run_timestamp": "2026-10-17T19:59:56.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.728333125, "accuracy_std_dev": 0.035862401001736835, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0596212475, "avg_duration_per_task_ms": 368463.805, "avg_output_tokens_per_task": 11783.2, "last_run_timestamp": "2026-10-17T20:00:29.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7279999, "accuracy_std_dev": 0.029784221028759515, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.29079123999999995, "avg_duration_per_task_ms": 357975.664, "avg_output_tokens_per_task": 9410.784, "last_run_timestamp": "2026-10-17T10:36:28.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.7277775, "accuracy_std_dev": 0.028652944605805083, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04601716333333333, "avg_duration_per_task_ms": 165703.69333333333, "avg_output_tokens_per_task": 8247.983333333334, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.7276191428571428, "accuracy_std_dev": 0.052906351183492316, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07805452, "avg_duration_per_task_ms": 228092.80571428573, "avg_output_tokens_per_task": 8318.434285714286, "last_run_timestamp": "2026-09-19T08:12:24.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.727316106017192, "accuracy_std_dev": 0.028771894791938337, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.1317654670487106, "avg_duration_per_task_ms": 71027.62464183381, "avg_output_tokens_per_task": 10119.80229226361, "last_run_timestamp": "2026-10-10T19:19:07.184Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.72722225, "accuracy_std_dev": 0.050365918751547674, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11962306833333335, "avg_duration_per_task_ms": 145630.83333333334, "avg_output_tokens_per_task": 9570.12, "last_run_timestamp": "2026-10-17T19:44:43.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7272221666666667, "accuracy_std_dev": 0.024863716190576896, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08010389666666666, "avg_duration_per_task_ms": 652312.8383333334, "avg_output_tokens_per_task": 8922.71, "last_run_timestamp": "2026-10-17T19:35:50.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7272220833333334, "accuracy_std_dev": 0.025012427159508587, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07207888500000001, "avg_duration_per_task_ms": 216694.285, "avg_output_tokens_per_task": 8508.088333333333, "last_run_timestamp": "2026-10-17T20:16:13.711Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11300302000000001, "avg_duration_per_task_ms": 353321.78, "avg_output_tokens_per_task": 3658.14, "last_run_timestamp": "2026-10-03T00:00:06.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7266667142857143, "accuracy_std_dev": 0.023637648207372713, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06641256285714285, "avg_duration_per_task_ms": 202688.7742857143, "avg_output_tokens_per_task": 9459.962857142857, "last_run_timestamp": "2026-10-17T19:21:20.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.7266665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11245736999999999, "avg_duration_per_task_ms": 138229.98, "avg_output_tokens_per_task": 17958.65, "last_run_timestamp": "2026-10-17T10:25:34.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7266665, "accuracy_std_dev": 0.03162256578378802, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.325251515, "avg_duration_per_task_ms": 113433.705, "avg_output_tokens_per_task": 10925.445, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7260606363636364, "accuracy_std_dev": 0.037546404958694955, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009626858181818184, "avg_duration_per_task_ms": 366334.8836363636, "avg_output_tokens_per_task": 14262.687272727273, "last_run_timestamp": "2026-10-17T13:13:33.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7260605454545455, "accuracy_std_dev": 0.03014507499459492, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.35443870000000005, "avg_duration_per_task_ms": 238811.5509090909, "avg_output_tokens_per_task": 10880.86, "last_run_timestamp": "2026-10-17T10:59:31.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7259886915254237, "accuracy_std_dev": 0.03421740126220705, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.01908770508474576, "avg_duration_per_task_ms": 310174.90847457625, "avg_output_tokens_per_task": 6632.271186440678, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7259258888888889, "accuracy_std_dev": 0.01817470468318503, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10746231555555555, "avg_duration_per_task_ms": 332961.9111111111, "avg_output_tokens_per_task": 14617.324444444444, "last_run_timestamp": "2026-10-17T19:16:53.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7255891616161616, "accuracy_std_dev": 0.0865306861288959, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.009908151515151516, "avg_duration_per_task_ms": 192785.0606060606, "avg_output_tokens_per_task": 4205.436868686868, "last_run_timestamp": "2026-10-17T11:01:14.130Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7250608065693431, "accuracy_std_dev": 0.027313728982127115, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.045819195255474454, "avg_duration_per_task_ms": 200398.98540145985, "avg_output_tokens_per_task": 9616.815693430657, "last_run_timestamp": "2026-10-17T19:49:20.976Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.725, "accuracy_std_dev": 0.025331238950157955, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.28661039000000005, "avg_duration_per_task_ms": 193369.4875, "avg_output_tokens_per_task": 11544.6525, "last_run_timestamp": "2026-10-17T11:22:12.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7248484545454545, "accuracy_std_dev": 0.026831392056739796, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.028918963636363638, "avg_duration_per_task_ms": 207327.25454545455, "avg_output_tokens_per_task": 9728.478181818182, "last_run_timestamp": "2026-10-17T20:29:55.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7244622983870967, "accuracy_std_dev": 0.027938338852551706, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.061894955645161294, "avg_duration_per_task_ms": 511647.25, "avg_output_tokens_per_task": 14459.54435483871, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7239058686868687, "accuracy_std_dev": 0.07830589921636386, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.044882882154882155, "avg_duration_per_task_ms": 272662.92087542085, "avg_output_tokens_per_task": 25044.632996632998, "last_run_timestamp": "2026-10-17T19:39:03.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.723680833976834, "accuracy_std_dev": 0.03403551347181265, "run_count": 26, "total_questions": 1295, "avg_cost_per_task_usd": 0.11716301235521236, "avg_duration_per_task_ms": 133519.80694980695, "avg_output_tokens_per_task": 8532.8, "last_run_timestamp": "2026-10-17T19:53:08.746Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05323206, "avg_duration_per_task_ms": 276804.85, "avg_output_tokens_per_task": 8575.93, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.72333325, "accuracy_std_dev": 0.01953163151627381, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08062917333333333, "avg_duration_per_task_ms": 245562.14, "avg_output_tokens_per_task": 9203.731666666667, "last_run_timestamp": "2026-10-17T19:20:50.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.72333325, "accuracy_std_dev": 0.01374358436462263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011429815, "avg_duration_per_task_ms": 101515.345, "avg_output_tokens_per_task": 5183.985, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.723137294117647, "accuracy_std_dev": 0.03345988135375875, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.07395268117647061, "avg_duration_per_task_ms": 120093.27058823529, "avg_output_tokens_per_task": 8763.595294117647, "last_run_timestamp": "2026-10-17T19:46:31.426Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7229628888888889, "accuracy_std_dev": 0.015748534890899425, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024130835555555553, "avg_duration_per_task_ms": 202737.1288888889, "avg_output_tokens_per_task": 9830.88, "last_run_timestamp": "2026-10-17T19:19:55.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.7224448316633266, "accuracy_std_dev": 0.04149011767939444, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.02709162324649299, "avg_duration_per_task_ms": 310028.67134268535, "avg_output_tokens_per_task": 6206.739478957916, "last_run_timestamp": "2026-10-17T11:01:17.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.7222221666666667, "accuracy_std_dev": 0.029228799430337363, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05327505, "avg_duration_per_task_ms": 275570.87333333335, "avg_output_tokens_per_task": 8563.536666666667, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7218181818181818, "accuracy_std_dev": 0.026981663672064338, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.042815734545454544, "avg_duration_per_task_ms": 128618.97818181818, "avg_output_tokens_per_task": 8570.907272727272, "last_run_timestamp": "2026-10-17T19:19:33.079Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7214815555555555, "accuracy_std_dev": 0.017149338413615003, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05164317555555556, "avg_duration_per_task_ms": 391354.9733333333, "avg_output_tokens_per_task": 9429.368888888888, "last_run_timestamp": "2026-10-17T20:23:07.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7213049198113207, "accuracy_std_dev": 0.02807943476656338, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.10996251533018866, "avg_duration_per_task_ms": 106559.05070754717, "avg_output_tokens_per_task": 9515.395047169812, "last_run_timestamp": "2026-10-17T19:37:10.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7207207364864865, "accuracy_std_dev": 0.029283397598582478, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.0602562027027027, "avg_duration_per_task_ms": 500890.8614864865, "avg_output_tokens_per_task": 14217.908783783783, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7205554166666667, "accuracy_std_dev": 0.06303779020484238, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.28826981500000004, "avg_duration_per_task_ms": 424234.8416666667, "avg_output_tokens_per_task": 10037.273333333333, "last_run_timestamp": "2026-10-17T11:17:27.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7205384141414142, "accuracy_std_dev": 0.051112796809483045, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 1.6761988181818182, "avg_duration_per_task_ms": 313695.66161616164, "avg_output_tokens_per_task": 23404.97474747475, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7203921764705883, "accuracy_std_dev": 0.017030604563326206, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.038154339999999995, "avg_duration_per_task_ms": 182427.35294117648, "avg_output_tokens_per_task": 9882.828235294117, "last_run_timestamp": "2026-10-17T19:25:39.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.72, "accuracy_std_dev": 0.03231762505197435, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.36211610499999997, "avg_duration_per_task_ms": 150104.23, "avg_output_tokens_per_task": 23123.4, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.053717240000000006, "avg_duration_per_task_ms": 263357.62, "avg_output_tokens_per_task": 8791.84, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.72, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06889947, "avg_duration_per_task_ms": 485555.13, "avg_output_tokens_per_task": 10139.72, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7199998888888889, "accuracy_std_dev": 0.02413946106111294, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06200596222222221, "avg_duration_per_task_ms": 305935.6088888889, "avg_output_tokens_per_task": 9573.94, "last_run_timestamp": "2026-10-17T19:52:16.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.71999975, "accuracy_std_dev": 0.03559033108426363, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2422260125, "avg_duration_per_task_ms": 293804.47, "avg_output_tokens_per_task": 21452.32, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7199996666666667, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09322211333333333, "avg_duration_per_task_ms": 149908.11333333334, "avg_output_tokens_per_task": 49346.81333333333, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7198066588628762, "accuracy_std_dev": 0.028976766258600285, "run_count": 18, "total_questions": 897, "avg_cost_per_task_usd": 0.4397325016722407, "avg_duration_per_task_ms": 240718.0635451505, "avg_output_tokens_per_task": 10095.885172798216, "last_run_timestamp": "2026-10-17T11:17:26.520Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7193332, "accuracy_std_dev": 0.03143937799893632, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045152332, "avg_duration_per_task_ms": 326196.928, "avg_output_tokens_per_task": 13427.836, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.7192592222222223, "accuracy_std_dev": 0.03561726435242629, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06394025333333334, "avg_duration_per_task_ms": 73154.98, "avg_output_tokens_per_task": 9365.82888888889, "last_run_timestamp": "2026-10-17T19:08:58.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.7188886666666667, "accuracy_std_dev": 0.010303924699949169, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4212243666666666, "avg_duration_per_task_ms": 138973.84666666668, "avg_output_tokens_per_task": 11088.173333333334, "last_run_timestamp": "2026-10-17T19:18:39.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7188551919191919, "accuracy_std_dev": 0.027525589315898408, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.29904142424242425, "avg_duration_per_task_ms": 154139.7323232323, "avg_output_tokens_per_task": 5036.570707070707, "last_run_timestamp": "2026-10-01T19:05:12.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.7185185555555555, "accuracy_std_dev": 0.02347109362651624, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07224364888888889, "avg_duration_per_task_ms": 300057.04, "avg_output_tokens_per_task": 10826.902222222223, "last_run_timestamp": "2026-10-17T22:09:39.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7175757272727272, "accuracy_std_dev": 0.03858842754244266, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10341143999999998, "avg_duration_per_task_ms": 109231.03818181818, "avg_output_tokens_per_task": 9277.165454545455, "last_run_timestamp": "2026-10-17T19:18:06.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7173648633879781, "accuracy_std_dev": 0.031638898989055704, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.0967067795992714, "avg_duration_per_task_ms": 190644.16393442624, "avg_output_tokens_per_task": 10605.225865209472, "last_run_timestamp": "2026-10-17T19:53:39.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.7170371111111111, "accuracy_std_dev": 0.016956299841680886, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12728777111111111, "avg_duration_per_task_ms": 101323.44222222222, "avg_output_tokens_per_task": 3885.56, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7169697272727272, "accuracy_std_dev": 0.03367855284920254, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12441471090909093, "avg_duration_per_task_ms": 167003.52181818182, "avg_output_tokens_per_task": 11951.263636363636, "last_run_timestamp": "2026-10-17T19:47:56.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7167631098265895, "accuracy_std_dev": 0.10367778899207734, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.07269199132947977, "avg_duration_per_task_ms": 149631.9450867052, "avg_output_tokens_per_task": 14661.630057803468, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015077799999999999, "avg_duration_per_task_ms": 476878.72, "avg_output_tokens_per_task": 26097.54, "last_run_timestamp": "2026-09-28T09:00:02.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7166665833333333, "accuracy_std_dev": 0.0346943826179838, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.038253503333333334, "avg_duration_per_task_ms": 190560.97666666665, "avg_output_tokens_per_task": 9125.601666666667, "last_run_timestamp": "2026-10-17T20:01:54.778Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7162160675675675, "accuracy_std_dev": 0.02984900053044771, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.1296834662162162, "avg_duration_per_task_ms": 769826.7162162162, "avg_output_tokens_per_task": 28396.02027027027, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7161111666666666, "accuracy_std_dev": 0.030847173070999906, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.04424468333333333, "avg_duration_per_task_ms": 144050.02166666667, "avg_output_tokens_per_task": 8482.466666666667, "last_run_timestamp": "2026-10-17T19:16:32.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7155553333333334, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13937678, "avg_duration_per_task_ms": 263819.3466666667, "avg_output_tokens_per_task": 10311.9, "last_run_timestamp": "2026-10-03T16:18:50.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7153846153846154, "accuracy_std_dev": 0.03837602219643538, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.08081019846153846, "avg_duration_per_task_ms": 94500.82769230769, "avg_output_tokens_per_task": 8578.88, "last_run_timestamp": "2026-10-17T19:41:19.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.07260889583333334, "avg_duration_per_task_ms": 556749.9791666666, "avg_output_tokens_per_task": 12329.4375, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.009582875, "avg_duration_per_task_ms": 164124.5, "avg_output_tokens_per_task": 18445.333333333332, "last_run_timestamp": "2026-10-10T11:44:02.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.715, "accuracy_std_dev": 0.029202378139459823, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09407665500000001, "avg_duration_per_task_ms": 83159.84, "avg_output_tokens_per_task": 3660.855, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7148146666666667, "accuracy_std_dev": 0.031075936024161484, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09082762666666666, "avg_duration_per_task_ms": 156907.88888888888, "avg_output_tokens_per_task": 48272.70222222222, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7144445, "accuracy_std_dev": 0.03515405426002334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13767769333333332, "avg_duration_per_task_ms": 330585.11666666664, "avg_output_tokens_per_task": 10602.806666666667, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7139639594594595, "accuracy_std_dev": 0.015199555307822523, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04553853378378378, "avg_duration_per_task_ms": 61304.33783783784, "avg_output_tokens_per_task": 8376.81081081081, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7133782062780268, "accuracy_std_dev": 0.10832537981105755, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.20968350448430492, "avg_duration_per_task_ms": 1204177.4775784754, "avg_output_tokens_per_task": 56724.81614349776, "last_run_timestamp": "2026-10-10T19:16:51.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05452906, "avg_duration_per_task_ms": 272872.18, "avg_output_tokens_per_task": 39719.44, "last_run_timestamp": "2026-10-13T20:01:29.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7128572857142856, "accuracy_std_dev": 0.02897348072050064, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.20664069142857142, "avg_duration_per_task_ms": 1520570.0285714285, "avg_output_tokens_per_task": 13765.78, "last_run_timestamp": "2026-10-17T19:37:45.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.712121101010101, "accuracy_std_dev": 0.029111223892169177, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12218062626262625, "avg_duration_per_task_ms": 537818.2474747475, "avg_output_tokens_per_task": 55422.06060606061, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.7121210774410774, "accuracy_std_dev": 0.067600587772757, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.05431976767676767, "avg_duration_per_task_ms": 1240970.0303030303, "avg_output_tokens_per_task": 60052.92929292929, "last_run_timestamp": "2026-10-17T11:01:17.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.71166675, "accuracy_std_dev": 0.019075733568790994, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07509325, "avg_duration_per_task_ms": 322731.32, "avg_output_tokens_per_task": 33483.435, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7113821300813009, "accuracy_std_dev": 0.04920965638205573, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.03321126422764228, "avg_duration_per_task_ms": 1038484.2195121951, "avg_output_tokens_per_task": 47360.833333333336, "last_run_timestamp": "2026-10-17T19:19:46.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.011331263428624743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07186812, "avg_duration_per_task_ms": 348295.26, "avg_output_tokens_per_task": 33945.17333333333, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.711111, "accuracy_std_dev": 0.014401748867566209, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02642942666666667, "avg_duration_per_task_ms": 207531.11555555556, "avg_output_tokens_per_task": 10062.235555555555, "last_run_timestamp": "2026-10-17T19:00:35.778Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7103705555555555, "accuracy_std_dev": 0.030809822424065814, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03036840888888889, "avg_duration_per_task_ms": 105409.86666666667, "avg_output_tokens_per_task": 5076.06, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12391315, "avg_duration_per_task_ms": 76197.77, "avg_output_tokens_per_task": 4134.025, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.709697, "accuracy_std_dev": 0.029489058220419185, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11235794000000002, "avg_duration_per_task_ms": 96345.32545454545, "avg_output_tokens_per_task": 9865.885454545454, "last_run_timestamp": "2026-10-17T19:23:11.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7093335999999999, "accuracy_std_dev": 0.027840743923968677, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.049385444, "avg_duration_per_task_ms": 869963.78, "avg_output_tokens_per_task": 8710.828, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.709172389261745, "accuracy_std_dev": 0.06451706767635644, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.07478329530201343, "avg_duration_per_task_ms": 78607.22147651007, "avg_output_tokens_per_task": 13899.020134228187, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.709166875, "accuracy_std_dev": 0.054968456232728356, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.030966977499999996, "avg_duration_per_task_ms": 308720.575, "avg_output_tokens_per_task": 10036.525, "last_run_timestamp": "2026-10-17T19:40:45.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7090909090909091, "accuracy_std_dev": 0.1551348146791596, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.058754954545454555, "avg_duration_per_task_ms": 149210.46545454545, "avg_output_tokens_per_task": 17057.438181818183, "last_run_timestamp": "2026-10-17T19:18:18.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7088888888888889, "accuracy_std_dev": 0.057777619658757995, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10292275111111111, "avg_duration_per_task_ms": 232970.33777777778, "avg_output_tokens_per_task": 10921.413333333334, "last_run_timestamp": "2026-10-17T19:18:15.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.01749773883169545, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11639628666666667, "avg_duration_per_task_ms": 755791.72, "avg_output_tokens_per_task": 27174.273333333334, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7075098814229249, "accuracy_std_dev": 0.03261134857329563, "run_count": 31, "total_questions": 1518, "avg_cost_per_task_usd": 0.09765357707509884, "avg_duration_per_task_ms": 73613.54281949934, "avg_output_tokens_per_task": 7991.078392621871, "last_run_timestamp": "2026-10-17T19:21:40.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7066664, "accuracy_std_dev": 0.01801227976242876, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.050093852, "avg_duration_per_task_ms": 393986.292, "avg_output_tokens_per_task": 8914.744, "last_run_timestamp": "2026-10-03T16:12:07.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7061769766277128, "accuracy_std_dev": 0.033226512089578226, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.07507928213689483, "avg_duration_per_task_ms": 218057.23372287146, "avg_output_tokens_per_task": 8906.173622704508, "last_run_timestamp": "2026-10-17T19:12:23.653Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.7060416875, "accuracy_std_dev": 0.03260073424540993, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.12466288875, "avg_duration_per_task_ms": 129451.30375, "avg_output_tokens_per_task": 8877.84875, "last_run_timestamp": "2026-10-17T19:20:16.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.7059999333333333, "accuracy_std_dev": 0.024104784834735375, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.13286914133333336, "avg_duration_per_task_ms": 86225.15866666667, "avg_output_tokens_per_task": 10631.385333333334, "last_run_timestamp": "2026-10-17T19:38:35.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7056452258064517, "accuracy_std_dev": 0.012405393094940622, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.6654760282258064, "avg_duration_per_task_ms": 686538.5927419355, "avg_output_tokens_per_task": 55521.342741935485, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.028782815862246673, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.059492744, "avg_duration_per_task_ms": 531413.344, "avg_output_tokens_per_task": 14170.264, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7037037777777777, "accuracy_std_dev": 0.0414682095178618, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06722512, "avg_duration_per_task_ms": 338497.55333333334, "avg_output_tokens_per_task": 9866.23111111111, "last_run_timestamp": "2026-10-17T20:25:29.287Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.7028112570281124, "accuracy_std_dev": 0.05430403649085398, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.21449976305220883, "avg_duration_per_task_ms": 647680.6907630522, "avg_output_tokens_per_task": 10064.048192771084, "last_run_timestamp": "2026-10-17T20:39:24.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.7016665000000001, "accuracy_std_dev": 0.005000166672222055, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.052099805000000006, "avg_duration_per_task_ms": 224630.845, "avg_output_tokens_per_task": 6764.525, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7012261270903011, "accuracy_std_dev": 0.02520740151666215, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.025383806020066887, "avg_duration_per_task_ms": 238193.83946488294, "avg_output_tokens_per_task": 9583.702341137123, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.701111, "accuracy_std_dev": 0.011811220865487748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13736028333333333, "avg_duration_per_task_ms": 494538.36333333334, "avg_output_tokens_per_task": 57774.223333333335, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.7000003333333334, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010915493333333333, "avg_duration_per_task_ms": 115952.82, "avg_output_tokens_per_task": 5345.16, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7000003333333332, "accuracy_std_dev": 0.024944382578492925, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12734470666666667, "avg_duration_per_task_ms": 591806.9066666666, "avg_output_tokens_per_task": 57532.753333333334, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.700000125, "accuracy_std_dev": 0.028674427245533173, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.047772974999999995, "avg_duration_per_task_ms": 357321.1875, "avg_output_tokens_per_task": 9223.68, "last_run_timestamp": "2026-10-17T20:08:34.955Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.019795359999999998, "avg_duration_per_task_ms": 396615.94, "avg_output_tokens_per_task": 23218.96, "last_run_timestamp": "2026-10-01T08:03:06.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6986667999999999, "accuracy_std_dev": 0.029028574428655626, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04290838, "avg_duration_per_task_ms": 173950.452, "avg_output_tokens_per_task": 8370.488, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6986607901785714, "accuracy_std_dev": 0.03366609083665969, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.045400854910714286, "avg_duration_per_task_ms": 512677.609375, "avg_output_tokens_per_task": 9135.091517857143, "last_run_timestamp": "2026-10-17T19:41:52.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.698333375, "accuracy_std_dev": 0.13007476749540772, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0438401425, "avg_duration_per_task_ms": 300334.5325, "avg_output_tokens_per_task": 10585.9875, "last_run_timestamp": "2026-10-17T21:09:43.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.6977779999999999, "accuracy_std_dev": 0.03624346343825322, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10053378, "avg_duration_per_task_ms": 541324.4933333333, "avg_output_tokens_per_task": 9996.04, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6977778333333333, "accuracy_std_dev": 0.017915990515148434, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21024033333333336, "avg_duration_per_task_ms": 335029.48333333334, "avg_output_tokens_per_task": 34166.46666666667, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.6975756363636364, "accuracy_std_dev": 0.042571053337114, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09368267636363638, "avg_duration_per_task_ms": 209598.28545454546, "avg_output_tokens_per_task": 8033.250909090909, "last_run_timestamp": "2026-10-17T19:56:11.216Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14710045000000002, "avg_duration_per_task_ms": 163276.98, "avg_output_tokens_per_task": 5053.91, "last_run_timestamp": "2026-10-01T08:02:59.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.6963602643678161, "accuracy_std_dev": 0.1081019669259783, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07385931609195404, "avg_duration_per_task_ms": 786132.5603448276, "avg_output_tokens_per_task": 45244.65804597701, "last_run_timestamp": "2026-10-17T19:13:10.419Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.6962752034383954, "accuracy_std_dev": 0.02843686742557259, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.019877143266475644, "avg_duration_per_task_ms": 515823.4040114613, "avg_output_tokens_per_task": 9119.95988538682, "last_run_timestamp": "2026-10-17T19:15:21.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6959998500000001, "accuracy_std_dev": 0.02293954179637203, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.10828757100000001, "avg_duration_per_task_ms": 77900.456, "avg_output_tokens_per_task": 9749.434, "last_run_timestamp": "2026-10-17T19:38:37.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6958144439461884, "accuracy_std_dev": 0.011732711186157182, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.02170840807174888, "avg_duration_per_task_ms": 570466.8318385651, "avg_output_tokens_per_task": 18922.206278026904, "last_run_timestamp": "2026-10-17T19:19:38.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.695, "accuracy_std_dev": 0.0016670000000000296, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11442463, "avg_duration_per_task_ms": 208708.95, "avg_output_tokens_per_task": 8898.45, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.69499975, "accuracy_std_dev": 0.041331999306681265, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.018806935000000004, "avg_duration_per_task_ms": 381666.575, "avg_output_tokens_per_task": 20477.295, "last_run_timestamp": "2026-10-17T20:55:29.683Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6949152745762711, "accuracy_std_dev": 0.028368895357513555, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.01705773898305085, "avg_duration_per_task_ms": 275649.04745762714, "avg_output_tokens_per_task": 9029.440677966102, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.054160228677918684, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02421678666666667, "avg_duration_per_task_ms": 198756.01333333334, "avg_output_tokens_per_task": 9626.628888888888, "last_run_timestamp": "2026-10-17T19:18:29.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.07159554093346564, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07348721999999999, "avg_duration_per_task_ms": 284784.05555555556, "avg_output_tokens_per_task": 26224.762222222223, "last_run_timestamp": "2026-10-17T19:45:29.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07614706, "avg_duration_per_task_ms": 539148.38, "avg_output_tokens_per_task": 13067.92, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01748764, "avg_duration_per_task_ms": 289873.3, "avg_output_tokens_per_task": 9630.44, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6923076113360324, "accuracy_std_dev": 0.004408577938519381, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.009435951417004049, "avg_duration_per_task_ms": 259361.54655870446, "avg_output_tokens_per_task": 9080.562753036438, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6917516734693877, "accuracy_std_dev": 0.023775708138770536, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.03340652295918367, "avg_duration_per_task_ms": 558953.0178571428, "avg_output_tokens_per_task": 18907.030612244896, "last_run_timestamp": "2026-10-17T19:21:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.6913265306122449, "accuracy_std_dev": 0.02634978866613541, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.024798336734693876, "avg_duration_per_task_ms": 305198.10204081633, "avg_output_tokens_per_task": 16421.35714285714, "last_run_timestamp": "2026-10-17T20:08:29.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6905556666666667, "accuracy_std_dev": 0.03202522925323443, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.035373304999999994, "avg_duration_per_task_ms": 164008.30833333332, "avg_output_tokens_per_task": 10108.841666666667, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.69, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01604401, "avg_duration_per_task_ms": 258586.65, "avg_output_tokens_per_task": 10109.57, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6891892297297297, "accuracy_std_dev": 0.03170486152346707, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6622049662162162, "avg_duration_per_task_ms": 777791.8783783783, "avg_output_tokens_per_task": 59355.4527027027, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.688485, "accuracy_std_dev": 0.07169463814489388, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11122534363636363, "avg_duration_per_task_ms": 72899.75090909092, "avg_output_tokens_per_task": 8778.474545454545, "last_run_timestamp": "2026-10-17T19:21:20.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6872909230769231, "accuracy_std_dev": 0.021147758508094125, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.006207953177257525, "avg_duration_per_task_ms": 252164.59866220737, "avg_output_tokens_per_task": 9519.789297658863, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.68666675, "accuracy_std_dev": 0.14596428025783398, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.093884975, "avg_duration_per_task_ms": 1131069.095, "avg_output_tokens_per_task": 24772.465, "last_run_timestamp": "2026-10-17T19:19:25.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.01943636340085252, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015455749999999999, "avg_duration_per_task_ms": 256063.75, "avg_output_tokens_per_task": 17914.69, "last_run_timestamp": "2026-10-17T20:59:34.683Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.07779388246014092, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00909372, "avg_duration_per_task_ms": 253755.2, "avg_output_tokens_per_task": 15941.146666666667, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.027148330554361683, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007905433333333333, "avg_duration_per_task_ms": 275517.85333333333, "avg_output_tokens_per_task": 12243.556666666667, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.683333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00634196, "avg_duration_per_task_ms": 404260.22, "avg_output_tokens_per_task": 17374, "last_run_timestamp": "2026-10-17T11:31:13.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.682, "accuracy_std_dev": 0.03438995977607416, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.017993061999999997, "avg_duration_per_task_ms": 315808.552, "avg_output_tokens_per_task": 20586.058, "last_run_timestamp": "2026-10-10T21:10:48.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6816665000000001, "accuracy_std_dev": 0.02327362599059285, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013877145, "avg_duration_per_task_ms": 70751.9, "avg_output_tokens_per_task": 5304.49, "last_run_timestamp": "2026-10-01T19:04:52.652Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6814285714285714, "accuracy_std_dev": 0.07495729245159184, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05172822285714287, "avg_duration_per_task_ms": 312143.8542857143, "avg_output_tokens_per_task": 8009.471428571429, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01975412, "avg_duration_per_task_ms": 454730.8, "avg_output_tokens_per_task": 9931.28, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6791665, "accuracy_std_dev": 0.017380712492587855, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.009063064999999999, "avg_duration_per_task_ms": 747112.36, "avg_output_tokens_per_task": 18370.14, "last_run_timestamp": "2026-10-17T19:40:56.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.6783334999999999, "accuracy_std_dev": 0.062070824243359295, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02936051, "avg_duration_per_task_ms": 178882.05, "avg_output_tokens_per_task": 10616.835, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.6761111666666667, "accuracy_std_dev": 0.020942326744471888, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017369979999999997, "avg_duration_per_task_ms": 120886.62666666666, "avg_output_tokens_per_task": 21445.503333333334, "last_run_timestamp": "2026-10-17T10:59:56.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.020245505523174504, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008486243333333334, "avg_duration_per_task_ms": 252988.28666666665, "avg_output_tokens_per_task": 9856.29, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.6734496104651163, "accuracy_std_dev": 0.08613577414329836, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.13526142151162793, "avg_duration_per_task_ms": 1917186.9389534884, "avg_output_tokens_per_task": 101904.42732558139, "last_run_timestamp": "2026-10-17T19:18:10.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64995528, "avg_duration_per_task_ms": 85231.23, "avg_output_tokens_per_task": 3880.06, "last_run_timestamp": "2026-10-01T08:05:26.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.02372680934208296, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11501549333333332, "avg_duration_per_task_ms": 145439.36666666667, "avg_output_tokens_per_task": 56059.84, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6733331428571429, "accuracy_std_dev": 0.014253843816705011, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.015046291428571428, "avg_duration_per_task_ms": 230830.76285714286, "avg_output_tokens_per_task": 10594.968571428572, "last_run_timestamp": "2026-10-13T22:01:15.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.04504326, "avg_duration_per_task_ms": 481727.7, "avg_output_tokens_per_task": 10202.42, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.672916875, "accuracy_std_dev": 0.0946110438485348, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.056751722500000004, "avg_duration_per_task_ms": 524628.53, "avg_output_tokens_per_task": 16401.87, "last_run_timestamp": "2026-10-10T19:25:46.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6724726434426229, "accuracy_std_dev": 0.11469794368845504, "run_count": 10, "total_questions": 488, "avg_cost_per_task_usd": 0.347430487704918, "avg_duration_per_task_ms": 1805676.3360655739, "avg_output_tokens_per_task": 58259.237704918036, "last_run_timestamp": "2026-10-17T11:24:43.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.671666625, "accuracy_std_dev": 0.0299072896180325, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.12573722500000004, "avg_duration_per_task_ms": 121720.524375, "avg_output_tokens_per_task": 12750.396875, "last_run_timestamp": "2026-10-17T11:08:18.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6706667, "accuracy_std_dev": 0.0663460470232402, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.061452046, "avg_duration_per_task_ms": 791003.158, "avg_output_tokens_per_task": 18308.462, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.670000125, "accuracy_std_dev": 0.08762293110601456, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09058643999999999, "avg_duration_per_task_ms": 103016.5525, "avg_output_tokens_per_task": 8462.2, "last_run_timestamp": "2026-10-17T19:21:30.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6690908181818183, "accuracy_std_dev": 0.03244538308329071, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11794495454545455, "avg_duration_per_task_ms": 125284.06909090909, "avg_output_tokens_per_task": 12716.46, "last_run_timestamp": "2026-10-17T10:30:47.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6666667246963563, "accuracy_std_dev": 0.034360494831273874, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.02177052024291498, "avg_duration_per_task_ms": 261725.5991902834, "avg_output_tokens_per_task": 16694.267206477733, "last_run_timestamp": "2026-10-17T19:37:21.849Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6657576363636364, "accuracy_std_dev": 0.02534973407166497, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11508544, "avg_duration_per_task_ms": 127141.62181818181, "avg_output_tokens_per_task": 12363.76909090909, "last_run_timestamp": "2026-10-17T10:30:02.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.6644445, "accuracy_std_dev": 0.0170691983091376, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.030060816666666667, "avg_duration_per_task_ms": 250207.05, "avg_output_tokens_per_task": 11154.95, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.664166625, "accuracy_std_dev": 0.11765935913362087, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05179430000000001, "avg_duration_per_task_ms": 715447.9325, "avg_output_tokens_per_task": 23872.2825, "last_run_timestamp": "2026-10-17T19:52:00.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.016629373093181295, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.026509213333333333, "avg_duration_per_task_ms": 393739.17, "avg_output_tokens_per_task": 10346.71, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.66166675, "accuracy_std_dev": 0.21009922457421754, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01252789, "avg_duration_per_task_ms": 392497.45, "avg_output_tokens_per_task": 10444.425, "last_run_timestamp": "2026-10-17T10:36:51.409Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.6616665, "accuracy_std_dev": 0.048333499999999974, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.006591860000000001, "avg_duration_per_task_ms": 431022.15, "avg_output_tokens_per_task": 20204.64, "last_run_timestamp": "2026-10-17T19:41:21.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6600001666666666, "accuracy_std_dev": 0.026666791666894545, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.023134516666666664, "avg_duration_per_task_ms": 309314.74333333335, "avg_output_tokens_per_task": 10345.606666666667, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.6596296666666667, "accuracy_std_dev": 0.10722601669267481, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09074199333333334, "avg_duration_per_task_ms": 219707.82, "avg_output_tokens_per_task": 22414.957777777778, "last_run_timestamp": "2026-10-17T19:35:52.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6594443333333334, "accuracy_std_dev": 0.015081891688010765, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02318081, "avg_duration_per_task_ms": 162431.44666666666, "avg_output_tokens_per_task": 17110.11, "last_run_timestamp": "2026-10-17T19:13:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.659393909090909, "accuracy_std_dev": 0.028349142244908185, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.12236485999999999, "avg_duration_per_task_ms": 121638.10242424242, "avg_output_tokens_per_task": 12651.811515151516, "last_run_timestamp": "2026-10-17T11:12:55.032Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6570882643678161, "accuracy_std_dev": 0.06117711592197373, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.024367669540229883, "avg_duration_per_task_ms": 460029.2816091954, "avg_output_tokens_per_task": 15624, "last_run_timestamp": "2026-10-17T19:35:41.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.656, "accuracy_std_dev": 0.02480152325160696, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022391364000000004, "avg_duration_per_task_ms": 255228.56, "avg_output_tokens_per_task": 9636.224, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6557823102040816, "accuracy_std_dev": 0.009318408855593308, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.014797057142857142, "avg_duration_per_task_ms": 277907.2775510204, "avg_output_tokens_per_task": 20245.579591836733, "last_run_timestamp": "2026-10-10T11:18:54.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.6555553333333334, "accuracy_std_dev": 0.04228062043326968, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.016284913333333335, "avg_duration_per_task_ms": 98025.37333333334, "avg_output_tokens_per_task": 20936.24, "last_run_timestamp": "2026-10-17T19:16:22.176Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.654166625, "accuracy_std_dev": 0.16981822491706944, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.13378775999999998, "avg_duration_per_task_ms": 366007.185, "avg_output_tokens_per_task": 8490.5075, "last_run_timestamp": "2026-10-17T20:38:51.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6536609479768786, "accuracy_std_dev": 0.0348083909646944, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.013188358381502889, "avg_duration_per_task_ms": 991739.563583815, "avg_output_tokens_per_task": 22797.29479768786, "last_run_timestamp": "2026-10-17T11:54:26.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.023726809342082957, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03210012, "avg_duration_per_task_ms": 143919.89333333334, "avg_output_tokens_per_task": 17353.273333333334, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6527314314381271, "accuracy_std_dev": 0.019242186339885847, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.01716779933110368, "avg_duration_per_task_ms": 303346.9264214047, "avg_output_tokens_per_task": 22796.782608695652, "last_run_timestamp": "2026-10-17T11:22:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.652196480620155, "accuracy_std_dev": 0.02424257709439149, "run_count": 13, "total_questions": 645, "avg_cost_per_task_usd": 0.010831201550387599, "avg_duration_per_task_ms": 324905.28992248065, "avg_output_tokens_per_task": 17153.43565891473, "last_run_timestamp": "2026-10-17T19:00:34.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6521739130434783, "accuracy_std_dev": 0.026636806048627927, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.014278464882943145, "avg_duration_per_task_ms": 801415.0635451504, "avg_output_tokens_per_task": 24999.354515050167, "last_run_timestamp": "2026-10-17T19:51:56.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.65166675, "accuracy_std_dev": 0.012801703448662586, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.031589305, "avg_duration_per_task_ms": 139715.76, "avg_output_tokens_per_task": 17049.6, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6515152727272728, "accuracy_std_dev": 0.024959225520075408, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11694987454545454, "avg_duration_per_task_ms": 144270.65272727274, "avg_output_tokens_per_task": 12693.569090909092, "last_run_timestamp": "2026-10-17T11:02:55.214Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.650067849593496, "accuracy_std_dev": 0.11323840378051966, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.025260294715447155, "avg_duration_per_task_ms": 965913.6768292683, "avg_output_tokens_per_task": 32745.57113821138, "last_run_timestamp": "2026-10-17T21:28:32.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6495238571428571, "accuracy_std_dev": 0.04088017277510516, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021405068571428574, "avg_duration_per_task_ms": 240437.21714285715, "avg_output_tokens_per_task": 27547.52857142857, "last_run_timestamp": "2026-10-17T10:59:26.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6485186666666666, "accuracy_std_dev": 0.03216903997807692, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03373100888888889, "avg_duration_per_task_ms": 73645.7311111111, "avg_output_tokens_per_task": 17406.14, "last_run_timestamp": "2026-10-17T19:37:10.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6477778333333334, "accuracy_std_dev": 0.03430091421724631, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09916395333333333, "avg_duration_per_task_ms": 153090.92333333334, "avg_output_tokens_per_task": 12859.136666666667, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.64666675, "accuracy_std_dev": 0.032317934478204226, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014356867499999997, "avg_duration_per_task_ms": 383641.005, "avg_output_tokens_per_task": 19373.5125, "last_run_timestamp": "2026-10-17T19:56:57.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.6465773571428571, "accuracy_std_dev": 0.12243509635005365, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.04724405580357143, "avg_duration_per_task_ms": 2135419.910714286, "avg_output_tokens_per_task": 77401.015625, "last_run_timestamp": "2026-10-17T11:23:30.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6464647272727273, "accuracy_std_dev": 0.040612499999999996, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.03484631313131314, "avg_duration_per_task_ms": 278588.8080808081, "avg_output_tokens_per_task": 18286.49494949495, "last_run_timestamp": "2026-10-17T19:18:50.561Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6455555000000001, "accuracy_std_dev": 0.022249880012785092, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.024322700000000003, "avg_duration_per_task_ms": 151214.55, "avg_output_tokens_per_task": 10419.556666666667, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6455555000000001, "accuracy_std_dev": 0.03206863203479896, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02137825, "avg_duration_per_task_ms": 227028.51, "avg_output_tokens_per_task": 10295.053333333333, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.644208, "accuracy_std_dev": 0.03904293739234052, "run_count": 3, "total_questions": 141, "avg_cost_per_task_usd": 0.024240567375886524, "avg_duration_per_task_ms": 70321.02836879433, "avg_output_tokens_per_task": 14782.985815602837, "last_run_timestamp": "2026-10-10T18:42:24.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6433335, "accuracy_std_dev": 0.014272415770172448, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031060726666666667, "avg_duration_per_task_ms": 229987.72666666665, "avg_output_tokens_per_task": 16358.803333333333, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6415203403508771, "accuracy_std_dev": 0.08180204086577275, "run_count": 6, "total_questions": 285, "avg_cost_per_task_usd": 0.12277472280701755, "avg_duration_per_task_ms": 351597.06315789477, "avg_output_tokens_per_task": 9359.947368421053, "last_run_timestamp": "2026-10-17T19:51:53.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6409490933014353, "accuracy_std_dev": 0.040131205785428226, "run_count": 17, "total_questions": 836, "avg_cost_per_task_usd": 0.020747283492822968, "avg_duration_per_task_ms": 318678.3744019139, "avg_output_tokens_per_task": 17826.28947368421, "last_run_timestamp": "2026-10-17T20:17:34.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.64083325, "accuracy_std_dev": 0.027525485666696242, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02212257, "avg_duration_per_task_ms": 261130.045, "avg_output_tokens_per_task": 10462.685, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.6396396317567568, "accuracy_std_dev": 0.06834614418790695, "run_count": 12, "total_questions": 592, "avg_cost_per_task_usd": 0.024531244932432434, "avg_duration_per_task_ms": 263890.6858108108, "avg_output_tokens_per_task": 25052.733108108107, "last_run_timestamp": "2026-10-17T19:25:29.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.04228062043326973, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08944130666666666, "avg_duration_per_task_ms": 118157.82666666666, "avg_output_tokens_per_task": 12612.513333333334, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.032810732962790655, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.20948366666666668, "avg_duration_per_task_ms": 978205.8133333334, "avg_output_tokens_per_task": 27155.76, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.6366665, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09132923, "avg_duration_per_task_ms": 148180.52, "avg_output_tokens_per_task": 3934.54, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.6333331666666667, "accuracy_std_dev": 0.10029589925551403, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017528723333333333, "avg_duration_per_task_ms": 260405.91333333333, "avg_output_tokens_per_task": 9076.7, "last_run_timestamp": "2026-10-17T19:37:16.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.633333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.024781250000000005, "avg_duration_per_task_ms": 171636.65, "avg_output_tokens_per_task": 11166.2, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.633333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.009703119999999999, "avg_duration_per_task_ms": 1596682.66, "avg_output_tokens_per_task": 19652.38, "last_run_timestamp": "2026-10-17T21:47:41.282Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.63, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02616634, "avg_duration_per_task_ms": 174261.43, "avg_output_tokens_per_task": 11763.02, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.63, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07672618, "avg_duration_per_task_ms": 231628.06, "avg_output_tokens_per_task": 10580.58, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6299998333333334, "accuracy_std_dev": 0.019531617294501993, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02836308333333333, "avg_duration_per_task_ms": 208192.55666666667, "avg_output_tokens_per_task": 10017.453333333333, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12269806, "avg_duration_per_task_ms": 610304.8666666667, "avg_output_tokens_per_task": 27046.16, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6239699483146067, "accuracy_std_dev": 0.026459500316288177, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.04252493033707865, "avg_duration_per_task_ms": 84689.76629213484, "avg_output_tokens_per_task": 23543.703370786516, "last_run_timestamp": "2026-10-17T19:09:20.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6233332500000001, "accuracy_std_dev": 0.038872984049819245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09536726749999998, "avg_duration_per_task_ms": 55598.6275, "avg_output_tokens_per_task": 3623.22, "last_run_timestamp": "2026-10-17T19:10:42.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6216667499999999, "accuracy_std_dev": 0.04997217837144482, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08547625, "avg_duration_per_task_ms": 106037.93, "avg_output_tokens_per_task": 11996.815, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.619999875, "accuracy_std_dev": 0.024036858272440176, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06868683249999999, "avg_duration_per_task_ms": 32416.59, "avg_output_tokens_per_task": 21460.735, "last_run_timestamp": "2026-10-17T19:10:22.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6196237419354839, "accuracy_std_dev": 0.011182496025485556, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.1178863306451613, "avg_duration_per_task_ms": 650732.7620967742, "avg_output_tokens_per_task": 30809.334677419356, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.619048, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.019175836734693877, "avg_duration_per_task_ms": 171955.81632653062, "avg_output_tokens_per_task": 9348.57142857143, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6076188571428572, "accuracy_std_dev": 0.03274303997331158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.046068719999999994, "avg_duration_per_task_ms": 191059.44, "avg_output_tokens_per_task": 3405.154285714286, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6060269955357143, "accuracy_std_dev": 0.032348691330028045, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.19772456473214287, "avg_duration_per_task_ms": 188670.52455357142, "avg_output_tokens_per_task": 13047.183035714286, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6006711006711409, "accuracy_std_dev": 0.09595263420158, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.03432377852348993, "avg_duration_per_task_ms": 2205358.8926174496, "avg_output_tokens_per_task": 69060.20469798657, "last_run_timestamp": "2026-10-17T20:16:20.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.59958325, "accuracy_std_dev": 0.026793203689322042, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.047474890000000006, "avg_duration_per_task_ms": 92900.3, "avg_output_tokens_per_task": 4620.69, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5943487528735633, "accuracy_std_dev": 0.021321805852608424, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.019723531609195398, "avg_duration_per_task_ms": 98994.94827586207, "avg_output_tokens_per_task": 4290.752873563219, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.5942761515151515, "accuracy_std_dev": 0.012018411802313968, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.15294240909090911, "avg_duration_per_task_ms": 508469.1313131313, "avg_output_tokens_per_task": 26213.378787878788, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.5935779504587155, "accuracy_std_dev": 0.242524807897738, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.017899249541284405, "avg_duration_per_task_ms": 294924.47155963304, "avg_output_tokens_per_task": 8919.59633027523, "last_run_timestamp": "2026-10-17T11:22:12.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.5933333333333334, "accuracy_std_dev": 0.02880349773127486, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21649743333333332, "avg_duration_per_task_ms": 191120.76666666666, "avg_output_tokens_per_task": 14122.313333333334, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.5918367959183675, "accuracy_std_dev": 0.0052774999999999905, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.09532348979591837, "avg_duration_per_task_ms": 107161.72448979592, "avg_output_tokens_per_task": 3646.7755102040815, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.5902778229166666, "accuracy_std_dev": 0.015776500000000027, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.01728564583333333, "avg_duration_per_task_ms": 131526.78125, "avg_output_tokens_per_task": 3837.7916666666665, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5881481111111111, "accuracy_std_dev": 0.007554077566224961, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029889999999999996, "avg_duration_per_task_ms": 78610.16, "avg_output_tokens_per_task": 8108.122222222222, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.584762, "accuracy_std_dev": 0.13245977235587955, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03989074857142857, "avg_duration_per_task_ms": 436437.8257142857, "avg_output_tokens_per_task": 8373.411428571428, "last_run_timestamp": "2026-10-10T19:10:33.654Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.58166675, "accuracy_std_dev": 0.11415155982809651, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.05917653, "avg_duration_per_task_ms": 650135.44, "avg_output_tokens_per_task": 38980.17, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5789978786127168, "accuracy_std_dev": 0.05180400838836742, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.18764681502890174, "avg_duration_per_task_ms": 2044873.4393063583, "avg_output_tokens_per_task": 41085.294797687864, "last_run_timestamp": "2026-10-17T19:42:10.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5762463812316716, "accuracy_std_dev": 0.04222508395322788, "run_count": 7, "total_questions": 341, "avg_cost_per_task_usd": 0.07726211143695015, "avg_duration_per_task_ms": 93864.66862170088, "avg_output_tokens_per_task": 5954.222873900293, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.5733334999999999, "accuracy_std_dev": 0.08212467486845838, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0333021775, "avg_duration_per_task_ms": 408902.2175, "avg_output_tokens_per_task": 17562.9, "last_run_timestamp": "2026-10-17T20:21:35.847Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.571111, "accuracy_std_dev": 0.008314764498569157, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028662713333333336, "avg_duration_per_task_ms": 78035.85333333333, "avg_output_tokens_per_task": 8001.593333333333, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5687678137535817, "accuracy_std_dev": 0.03326857759790391, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.011248378223495703, "avg_duration_per_task_ms": 234594.50429799428, "avg_output_tokens_per_task": 18173.54441260745, "last_run_timestamp": "2026-10-17T19:50:06.036Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.5622223333333333, "accuracy_std_dev": 0.03456997912768952, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.23936231333333335, "avg_duration_per_task_ms": 165980.19333333333, "avg_output_tokens_per_task": 13303.673333333334, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.5590475714285714, "accuracy_std_dev": 0.10244195347729805, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03983742857142857, "avg_duration_per_task_ms": 100656.9, "avg_output_tokens_per_task": 4373.865714285715, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5511113333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.035753513333333334, "avg_duration_per_task_ms": 61316.40666666667, "avg_output_tokens_per_task": 6659.566666666667, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5438595101214574, "accuracy_std_dev": 0.02876780459472013, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.002696097165991903, "avg_duration_per_task_ms": 431895.5668016194, "avg_output_tokens_per_task": 6592.368421052632, "last_run_timestamp": "2026-10-17T19:51:20.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5387596395348836, "accuracy_std_dev": 0.016016421054272853, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.021050491279069765, "avg_duration_per_task_ms": 1421665.261627907, "avg_output_tokens_per_task": 95835.8953488372, "last_run_timestamp": "2026-10-17T19:40:18.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.5360134020100502, "accuracy_std_dev": 0.08239086668587728, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.0516702864321608, "avg_duration_per_task_ms": 428067, "avg_output_tokens_per_task": 8053.70351758794, "last_run_timestamp": "2026-10-17T19:13:31.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5333335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10219375, "avg_duration_per_task_ms": 375443.12, "avg_output_tokens_per_task": 17495.06, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5282739330357142, "accuracy_std_dev": 0.019775495122330913, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.06609532812499999, "avg_duration_per_task_ms": 286684.00223214284, "avg_output_tokens_per_task": 4048.9084821428573, "last_run_timestamp": "2026-10-17T19:35:54.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5271991666666667, "accuracy_std_dev": 0.10370817001031737, "run_count": 6, "total_questions": 288, "avg_cost_per_task_usd": 0.062181392361111115, "avg_duration_per_task_ms": 408554.8958333333, "avg_output_tokens_per_task": 26661.76388888889, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5221089591836734, "accuracy_std_dev": 0.023889008586586408, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.13644156632653062, "avg_duration_per_task_ms": 73125.92346938775, "avg_output_tokens_per_task": 2553.7397959183672, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.5188173145161291, "accuracy_std_dev": 0.2566565496323832, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.007278975806451614, "avg_duration_per_task_ms": 482880.30241935485, "avg_output_tokens_per_task": 15769.91935483871, "last_run_timestamp": "2026-10-10T18:54:19.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517500125, "accuracy_std_dev": 0.024930426803594342, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0282700725, "avg_duration_per_task_ms": 444815, "avg_output_tokens_per_task": 58343.125, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.5173608125000001, "accuracy_std_dev": 0.025072499999999998, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.06539028125, "avg_duration_per_task_ms": 2881049.1770833335, "avg_output_tokens_per_task": 39867.583333333336, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5156599127516779, "accuracy_std_dev": 0.03449671802746843, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.024214489932885906, "avg_duration_per_task_ms": 190286.47651006712, "avg_output_tokens_per_task": 48937.10738255033, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.5104164999999999, "accuracy_std_dev": 0.003472500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.024820281250000003, "avg_duration_per_task_ms": 480070.4583333333, "avg_output_tokens_per_task": 9578.8125, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.5099998888888889, "accuracy_std_dev": 0.03456977556718343, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02963195333333334, "avg_duration_per_task_ms": 260966.97555555555, "avg_output_tokens_per_task": 20350.56, "last_run_timestamp": "2026-10-17T19:36:01.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5083332500000001, "accuracy_std_dev": 0.026404802805313633, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12635733, "avg_duration_per_task_ms": 53878.245, "avg_output_tokens_per_task": 2800.47, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5051021836734694, "accuracy_std_dev": 0.011159046957513885, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.019562938775510203, "avg_duration_per_task_ms": 1197227.9285714286, "avg_output_tokens_per_task": 100212.39285714286, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5033670303030303, "accuracy_std_dev": 0.030340499999999992, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.01769851515151515, "avg_duration_per_task_ms": 135563.53535353535, "avg_output_tokens_per_task": 30988.575757575756, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.5033669747474748, "accuracy_std_dev": 0.027292446564651823, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.025276146464646466, "avg_duration_per_task_ms": 577739.3712121212, "avg_output_tokens_per_task": 53598.469696969696, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.4995211379310344, "accuracy_std_dev": 0.03521937973703148, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.16297116666666667, "avg_duration_per_task_ms": 2016445.1580459771, "avg_output_tokens_per_task": 38482.971264367814, "last_run_timestamp": "2026-10-17T19:35:49.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23109727999999996, "avg_duration_per_task_ms": 80168.37, "avg_output_tokens_per_task": 2746.03, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48833333333333334, "accuracy_std_dev": 0.018831396165859726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03323320666666667, "avg_duration_per_task_ms": 665027.6466666666, "avg_output_tokens_per_task": 61694.416666666664, "last_run_timestamp": "2026-10-03T16:04:32.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.4852150080645161, "accuracy_std_dev": 0.023420064714684302, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.028619552419354842, "avg_duration_per_task_ms": 386610.2439516129, "avg_output_tokens_per_task": 66355.3125, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.4850950081300813, "accuracy_std_dev": 0.026197876665104, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.02379429674796748, "avg_duration_per_task_ms": 820057.8048780488, "avg_output_tokens_per_task": 22035.906504065042, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.48266679999999995, "accuracy_std_dev": 0.013727319074021705, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.067269828, "avg_duration_per_task_ms": 279626.944, "avg_output_tokens_per_task": 4093.296, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47874697986577186, "accuracy_std_dev": 0.00975008705374242, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.16965920805369125, "avg_duration_per_task_ms": 2424316.9261744968, "avg_output_tokens_per_task": 45379.409395973154, "last_run_timestamp": "2026-10-17T19:38:30.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.4777776666666667, "accuracy_std_dev": 0.013698571612479242, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12510532, "avg_duration_per_task_ms": 53067.566666666666, "avg_output_tokens_per_task": 2709.2933333333335, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.4755555, "accuracy_std_dev": 0.023306820916847493, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03474162, "avg_duration_per_task_ms": 44831.98333333333, "avg_output_tokens_per_task": 3658.7366666666667, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.47333357142857135, "accuracy_std_dev": 0.03837997162902674, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02437984, "avg_duration_per_task_ms": 61492.5, "avg_output_tokens_per_task": 3778.7314285714288, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.47222181249999995, "accuracy_std_dev": 0.0011595000000000077, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.07886530208333332, "avg_duration_per_task_ms": 55709.895833333336, "avg_output_tokens_per_task": 3493.7291666666665, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.4707204932432433, "accuracy_std_dev": 0.04373885161863152, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.028365344594594593, "avg_duration_per_task_ms": 2449502.6824324327, "avg_output_tokens_per_task": 141559.9391891892, "last_run_timestamp": "2026-10-17T19:53:16.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.47047614285714284, "accuracy_std_dev": 0.0261516387382106, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.025336362857142858, "avg_duration_per_task_ms": 119394.00571428571, "avg_output_tokens_per_task": 3796.2971428571427, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.467619, "accuracy_std_dev": 0.035173696625258384, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08029666571428572, "avg_duration_per_task_ms": 101903.78, "avg_output_tokens_per_task": 2752.694285714286, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4672696511627907, "accuracy_std_dev": 0.037956496396985506, "run_count": 8, "total_questions": 387, "avg_cost_per_task_usd": 0.0519110826873385, "avg_duration_per_task_ms": 199894.31782945737, "avg_output_tokens_per_task": 89805.21447028423, "last_run_timestamp": "2026-10-17T18:58:49.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.466667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03175828, "avg_duration_per_task_ms": 435844.14, "avg_output_tokens_per_task": 22556.3, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.4666666666666667, "accuracy_std_dev": 0.019626040903747127, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020897343333333335, "avg_duration_per_task_ms": 623575.84, "avg_output_tokens_per_task": 20727.386666666665, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.4662164729729729, "accuracy_std_dev": 0.03892660093903227, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04219703378378378, "avg_duration_per_task_ms": 711257.6756756756, "avg_output_tokens_per_task": 98782.22297297297, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.4650001666666666, "accuracy_std_dev": 0.03531613575226233, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.018810040000000004, "avg_duration_per_task_ms": 93827.55333333333, "avg_output_tokens_per_task": 3599.63, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46416674999999996, "accuracy_std_dev": 0.03386045095590872, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.022192442499999996, "avg_duration_per_task_ms": 225702.4075, "avg_output_tokens_per_task": 3591.99, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.46166650000000004, "accuracy_std_dev": 0.008333499999999994, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.035224570000000004, "avg_duration_per_task_ms": 486853.18, "avg_output_tokens_per_task": 23253.82, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4608501677852349, "accuracy_std_dev": 0.034994868451198195, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04671855033557047, "avg_duration_per_task_ms": 767600.1241610738, "avg_output_tokens_per_task": 101728.0033557047, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.46068351282051284, "accuracy_std_dev": 0.04225414673955088, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.01900871794871795, "avg_duration_per_task_ms": 170098.1794871795, "avg_output_tokens_per_task": 35854.89743589744, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.45666666666666667, "accuracy_std_dev": 0.03880952218778845, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.053214168888888885, "avg_duration_per_task_ms": 167547.09333333332, "avg_output_tokens_per_task": 3746.711111111111, "last_run_timestamp": "2026-10-17T19:11:29.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4563758489932886, "accuracy_std_dev": 0.0483687203813925, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.015332644295302013, "avg_duration_per_task_ms": 126207.14429530202, "avg_output_tokens_per_task": 3312.3657718120803, "last_run_timestamp": "2026-10-17T19:19:30.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.453333375, "accuracy_std_dev": 0.04533822277597982, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0537951275, "avg_duration_per_task_ms": 125879.785, "avg_output_tokens_per_task": 3538.1725, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.45183054913294796, "accuracy_std_dev": 0.04493510200731899, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.05361997109826589, "avg_duration_per_task_ms": 73759.52312138729, "avg_output_tokens_per_task": 3369.815028901734, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.4516665, "accuracy_std_dev": 0.02954922760270845, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.021153063333333333, "avg_duration_per_task_ms": 604099.4133333333, "avg_output_tokens_per_task": 21039.02666666667, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4511785656565656, "accuracy_std_dev": 0.03515039216708685, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04521708585858586, "avg_duration_per_task_ms": 779859.6717171717, "avg_output_tokens_per_task": 105433.07575757576, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.44708325000000004, "accuracy_std_dev": 0.03075165420245715, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.040560375, "avg_duration_per_task_ms": 277235.58, "avg_output_tokens_per_task": 3614.425, "last_run_timestamp": "2026-10-10T19:42:35.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.44333325, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025700495, "avg_duration_per_task_ms": 61023.305, "avg_output_tokens_per_task": 2441.375, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.4428572857142857, "accuracy_std_dev": 0.06825330771202708, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.17141447999999998, "avg_duration_per_task_ms": 117506.95428571428, "avg_output_tokens_per_task": 3352.2, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4416665, "accuracy_std_dev": 0.017873086533948188, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.104832345, "avg_duration_per_task_ms": 890298.59, "avg_output_tokens_per_task": 21334.155, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.43999985714285716, "accuracy_std_dev": 0.02225407399895624, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.036167245714285715, "avg_duration_per_task_ms": 123578.75714285714, "avg_output_tokens_per_task": 3292.997142857143, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.4391890202702703, "accuracy_std_dev": 0.03559610686773047, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.054588496621621616, "avg_duration_per_task_ms": 244762.44932432432, "avg_output_tokens_per_task": 2182.847972972973, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4383965367088608, "accuracy_std_dev": 0.03960836597472182, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.05240046329113924, "avg_duration_per_task_ms": 422067.2658227848, "avg_output_tokens_per_task": 20533.427848101266, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.43777799999999994, "accuracy_std_dev": 0.03861798340151904, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.19450383999999998, "avg_duration_per_task_ms": 50798.87333333334, "avg_output_tokens_per_task": 2047.5333333333333, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.011330936570096718, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02595394666666667, "avg_duration_per_task_ms": 66273.3, "avg_output_tokens_per_task": 2407.2933333333335, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21387673999999998, "avg_duration_per_task_ms": 47267.2, "avg_output_tokens_per_task": 2224.56, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.4362416845637584, "accuracy_std_dev": 0.024054442567272728, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.016195271812080535, "avg_duration_per_task_ms": 219855.4865771812, "avg_output_tokens_per_task": 2778.412751677852, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.4358108783783784, "accuracy_std_dev": 0.3084304226783661, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.015068885135135136, "avg_duration_per_task_ms": 91458.4831081081, "avg_output_tokens_per_task": 15700.83445945946, "last_run_timestamp": "2026-10-10T18:32:50.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.435, "accuracy_std_dev": 0.020749805890658335, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.074401825, "avg_duration_per_task_ms": 778012.545, "avg_output_tokens_per_task": 20313.29, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.43434374747474747, "accuracy_std_dev": 0.012448999999999988, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.023558444444444444, "avg_duration_per_task_ms": 1871236.9090909092, "avg_output_tokens_per_task": 14653.222222222223, "last_run_timestamp": "2026-10-13T10:40:58.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.43434354545454545, "accuracy_std_dev": 0.022037645087440712, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.06906736868686869, "avg_duration_per_task_ms": 323353.22222222225, "avg_output_tokens_per_task": 3378.3585858585857, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43416662500000003, "accuracy_std_dev": 0.01613729505630899, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2170375225, "avg_duration_per_task_ms": 46329.7025, "avg_output_tokens_per_task": 2265.4175, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4337431393442623, "accuracy_std_dev": 0.021677717946315293, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.032824139344262294, "avg_duration_per_task_ms": 478154.40573770495, "avg_output_tokens_per_task": 4694.745901639344, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.43333342857142854, "accuracy_std_dev": 0.04507497105888349, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4237074657142857, "avg_duration_per_task_ms": 109389.9, "avg_output_tokens_per_task": 3189.074285714286, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.4313334, "accuracy_std_dev": 0.02362666666798345, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.027534839999999994, "avg_duration_per_task_ms": 80491.464, "avg_output_tokens_per_task": 2779.172, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.431111, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029769013333333334, "avg_duration_per_task_ms": 630152.1266666667, "avg_output_tokens_per_task": 18760.18666666667, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.43089426016260163, "accuracy_std_dev": 0.03483555910043644, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.04202976422764227, "avg_duration_per_task_ms": 58240.52032520325, "avg_output_tokens_per_task": 2134.682926829268, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.43083325, "accuracy_std_dev": 0.031213896924086562, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029063365, "avg_duration_per_task_ms": 761962.965, "avg_output_tokens_per_task": 19001.86, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.24811575000000002, "avg_duration_per_task_ms": 46287.18, "avg_output_tokens_per_task": 2565.8, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.4293334, "accuracy_std_dev": 0.013727396775791108, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046574660000000004, "avg_duration_per_task_ms": 138519.112, "avg_output_tokens_per_task": 2139.824, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.4238095714285714, "accuracy_std_dev": 0.013384183649337146, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.024649345714285717, "avg_duration_per_task_ms": 219498.08857142858, "avg_output_tokens_per_task": 3556.6628571428573, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.423333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03482589, "avg_duration_per_task_ms": 291563.16, "avg_output_tokens_per_task": 21648.1, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.4222225, "accuracy_std_dev": 0.04331906317758191, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05459885, "avg_duration_per_task_ms": 144935.76666666666, "avg_output_tokens_per_task": 3474.9266666666667, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.42196971363636365, "accuracy_std_dev": 0.02414578127403073, "run_count": 9, "total_questions": 440, "avg_cost_per_task_usd": 0.03133452727272727, "avg_duration_per_task_ms": 644947.5272727272, "avg_output_tokens_per_task": 6264.340909090909, "last_run_timestamp": "2026-10-17T19:21:15.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.42006044796380093, "accuracy_std_dev": 0.026085942399861443, "run_count": 9, "total_questions": 442, "avg_cost_per_task_usd": 0.05284868552036199, "avg_duration_per_task_ms": 192763.79411764705, "avg_output_tokens_per_task": 4184.2692307692305, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.42, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02779026, "avg_duration_per_task_ms": 140935.14, "avg_output_tokens_per_task": 10384.88, "last_run_timestamp": "2026-10-17T11:21:49.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.41901883467741935, "accuracy_std_dev": 0.027582004591581083, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.02511988508064516, "avg_duration_per_task_ms": 390541.47782258067, "avg_output_tokens_per_task": 3601.5826612903224, "last_run_timestamp": "2026-10-17T19:23:43.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.40897435897435896, "accuracy_std_dev": 0.021673182250407416, "run_count": 8, "total_questions": 390, "avg_cost_per_task_usd": 0.011223169230769232, "avg_duration_per_task_ms": 108184.48717948717, "avg_output_tokens_per_task": 3251.0871794871796, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.408060476070529, "accuracy_std_dev": 0.1308846142804411, "run_count": 16, "total_questions": 794, "avg_cost_per_task_usd": 0.039315931989924426, "avg_duration_per_task_ms": 629857.338790932, "avg_output_tokens_per_task": 160922.7934508816, "last_run_timestamp": "2026-10-17T19:35:54.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40777783333333334, "accuracy_std_dev": 0.01781244190087991, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025491406666666664, "avg_duration_per_task_ms": 73232.91333333333, "avg_output_tokens_per_task": 2860.7966666666666, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.4053599698492462, "accuracy_std_dev": 0.008101744842933528, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.06672347236180905, "avg_duration_per_task_ms": 212067.50753768845, "avg_output_tokens_per_task": 8080.718592964824, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.40333342857142856, "accuracy_std_dev": 0.021894458241150196, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06258026857142857, "avg_duration_per_task_ms": 99625.90285714286, "avg_output_tokens_per_task": 2910.902857142857, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.4019045714285715, "accuracy_std_dev": 0.02611661099134934, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07661166857142858, "avg_duration_per_task_ms": 119529.87428571428, "avg_output_tokens_per_task": 3108.5485714285714, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.4, "accuracy_std_dev": 0.03126941809500138, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06669086285714285, "avg_duration_per_task_ms": 316867.28285714285, "avg_output_tokens_per_task": 12448.365714285714, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.39809542857142854, "accuracy_std_dev": 0.024355591348993847, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.027166705714285717, "avg_duration_per_task_ms": 311043.09428571427, "avg_output_tokens_per_task": 4197.714285714285, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.39809514285714287, "accuracy_std_dev": 0.03431195953278671, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06120218857142856, "avg_duration_per_task_ms": 337486.8342857143, "avg_output_tokens_per_task": 8320.111428571428, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.3980540203045685, "accuracy_std_dev": 0.06137496179721317, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.004494365482233502, "avg_duration_per_task_ms": 319107.7182741117, "avg_output_tokens_per_task": 4432.3045685279185, "last_run_timestamp": "2026-10-17T19:41:17.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.3975043048128342, "accuracy_std_dev": 0.017113317027610396, "run_count": 4, "total_questions": 187, "avg_cost_per_task_usd": 0.008529352941176469, "avg_duration_per_task_ms": 72525.10695187165, "avg_output_tokens_per_task": 1231.951871657754, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3970371111111111, "accuracy_std_dev": 0.12102898971407583, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.023592877777777778, "avg_duration_per_task_ms": 195708.14444444445, "avg_output_tokens_per_task": 3363.0533333333333, "last_run_timestamp": "2026-10-17T19:58:01.087Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.396667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0528661, "avg_duration_per_task_ms": 135118.22, "avg_output_tokens_per_task": 7825.32, "last_run_timestamp": "2026-07-25T08:07:19.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.3961905714285714, "accuracy_std_dev": 0.05390861756411, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04983603428571428, "avg_duration_per_task_ms": 88538.82571428572, "avg_output_tokens_per_task": 3736.18, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.3944446666666666, "accuracy_std_dev": 0.015947545189011245, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07351260000000001, "avg_duration_per_task_ms": 297595.3333333333, "avg_output_tokens_per_task": 7950.953333333333, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3942857142857143, "accuracy_std_dev": 0.02060319637278425, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.025239111428571426, "avg_duration_per_task_ms": 231992.80571428573, "avg_output_tokens_per_task": 4356.64, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.39351850505050506, "accuracy_std_dev": 0.030148094697001596, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.05853486616161616, "avg_duration_per_task_ms": 386334.952020202, "avg_output_tokens_per_task": 3836.3636363636365, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.39210027167630057, "accuracy_std_dev": 0.08499299536756556, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.3060756098265896, "avg_duration_per_task_ms": 324526.6416184971, "avg_output_tokens_per_task": 12470.086705202311, "last_run_timestamp": "2026-10-10T18:31:45.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.39047600000000005, "accuracy_std_dev": 0.017036713725699232, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03704184285714286, "avg_duration_per_task_ms": 191510.86285714287, "avg_output_tokens_per_task": 3720.674285714286, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.39, "accuracy_std_dev": 0.014782396941407474, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028387520000000006, "avg_duration_per_task_ms": 78569.82333333333, "avg_output_tokens_per_task": 4258.516666666666, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.3877778333333333, "accuracy_std_dev": 0.00895805727481629, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.014963976666666667, "avg_duration_per_task_ms": 92139.43333333333, "avg_output_tokens_per_task": 608.4966666666667, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.38583349999999994, "accuracy_std_dev": 0.06396070625391811, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.006338907499999999, "avg_duration_per_task_ms": 153451.2725, "avg_output_tokens_per_task": 3929.855, "last_run_timestamp": "2026-10-10T19:09:43.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.372222, "accuracy_std_dev": 0.019019243018585152, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03316533666666667, "avg_duration_per_task_ms": 160394.92, "avg_output_tokens_per_task": 4455.986666666667, "last_run_timestamp": "2026-10-03T16:07:02.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.37148306580829754, "accuracy_std_dev": 0.022649811310652534, "run_count": 14, "total_questions": 699, "avg_cost_per_task_usd": 0.01332730329041488, "avg_duration_per_task_ms": 37995.6452074392, "avg_output_tokens_per_task": 713.623748211731, "last_run_timestamp": "2026-10-17T19:12:18.209Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.3695237142857143, "accuracy_std_dev": 0.029996155461822036, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05283062, "avg_duration_per_task_ms": 74136.42571428571, "avg_output_tokens_per_task": 4382.251428571429, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.35925922222222223, "accuracy_std_dev": 0.032270942308928206, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08270178888888888, "avg_duration_per_task_ms": 25491.448888888888, "avg_output_tokens_per_task": 960.1, "last_run_timestamp": "2026-10-17T18:14:44.847Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35752687096774194, "accuracy_std_dev": 0.02846485415894485, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.07167372177419354, "avg_duration_per_task_ms": 382176.28830645164, "avg_output_tokens_per_task": 7777.157258064516, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.3519345982142857, "accuracy_std_dev": 0.02099059885040837, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07758646428571427, "avg_duration_per_task_ms": 205456.72098214287, "avg_output_tokens_per_task": 7279.627232142857, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.348333, "accuracy_std_dev": 0.015000000000000013, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.042842359999999996, "avg_duration_per_task_ms": 116081.37, "avg_output_tokens_per_task": 4431.82, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.333333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02900084, "avg_duration_per_task_ms": 82898.03, "avg_output_tokens_per_task": 3525.09, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.32800009999999996, "accuracy_std_dev": 0.024367614210053474, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08013712, "avg_duration_per_task_ms": 148773.196, "avg_output_tokens_per_task": 8016.85, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.32660695515695065, "accuracy_std_dev": 0.04329336143048513, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.03824817264573991, "avg_duration_per_task_ms": 96365.55156950673, "avg_output_tokens_per_task": 1277.7242152466367, "last_run_timestamp": "2026-10-17T18:24:53.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3093607534246575, "accuracy_std_dev": 0.03377945099423942, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.014691332191780824, "avg_duration_per_task_ms": 145559.72602739726, "avg_output_tokens_per_task": 1356.4246575342465, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.30916662500000003, "accuracy_std_dev": 0.23192285519970726, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0994847125, "avg_duration_per_task_ms": 229836.265, "avg_output_tokens_per_task": 28937.29, "last_run_timestamp": "2026-10-17T19:13:29.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.29925922222222223, "accuracy_std_dev": 0.03737628995552537, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.17081659111111108, "avg_duration_per_task_ms": 141383.24444444446, "avg_output_tokens_per_task": 1902.9044444444444, "last_run_timestamp": "2026-10-17T19:19:08.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.28901757803468203, "accuracy_std_dev": 0.24326737555175115, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.01199693063583815, "avg_duration_per_task_ms": 1302513.62716763, "avg_output_tokens_per_task": 80546.70520231214, "last_run_timestamp": "2026-10-17T19:19:57.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.2844443333333333, "accuracy_std_dev": 0.031894788459698065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02081186, "avg_duration_per_task_ms": 62184.12666666666, "avg_output_tokens_per_task": 3804.7533333333336, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.28000025, "accuracy_std_dev": 0.027080189561513406, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02535222, "avg_duration_per_task_ms": 67904.845, "avg_output_tokens_per_task": 4023.05, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.2683333333333333, "accuracy_std_dev": 0.3228473030880423, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.016053078333333335, "avg_duration_per_task_ms": 49421.746666666666, "avg_output_tokens_per_task": 8722.081666666667, "last_run_timestamp": "2026-10-17T19:48:24.252Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2622223333333333, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08337879999999999, "avg_duration_per_task_ms": 576772.3666666667, "avg_output_tokens_per_task": 5220.566666666667, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.25879402512562816, "accuracy_std_dev": 0.19158936601948973, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.1755232135678392, "avg_duration_per_task_ms": 1051691.3015075377, "avg_output_tokens_per_task": 6720.771356783919, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24200919178082192, "accuracy_std_dev": 0.027381797668443095, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.07592458219178082, "avg_duration_per_task_ms": 576526.0342465753, "avg_output_tokens_per_task": 5262.068493150685, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.22095242857142855, "accuracy_std_dev": 0.06555531427048489, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.012673305714285711, "avg_duration_per_task_ms": 20818.634285714284, "avg_output_tokens_per_task": 1052.7457142857143, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.2066666, "accuracy_std_dev": 0.10724846748480837, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.19412482799999997, "avg_duration_per_task_ms": 581641.784, "avg_output_tokens_per_task": 11075.26, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.19666649999999997, "accuracy_std_dev": 0.023804971471312466, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00425299, "avg_duration_per_task_ms": 15962.445, "avg_output_tokens_per_task": 1579.045, "last_run_timestamp": "2026-10-03T16:07:44.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.18766967073170732, "accuracy_std_dev": 0.018396157170452745, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.011055426829268291, "avg_duration_per_task_ms": 138140.34552845528, "avg_output_tokens_per_task": 5390.59756097561, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.18666649999999999, "accuracy_std_dev": 0.13029887012397048, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01464795, "avg_duration_per_task_ms": 65215.473333333335, "avg_output_tokens_per_task": 1173.1066666666666, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.13333333333333333, "accuracy_std_dev": 0.021430473285072877, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007961373333333334, "avg_duration_per_task_ms": 80728.30333333333, "avg_output_tokens_per_task": 2469.17, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.11833325, "accuracy_std_dev": 0.018484159885899604, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007158584999999999, "avg_duration_per_task_ms": 55526.67, "avg_output_tokens_per_task": 2353.445, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.108889, "accuracy_std_dev": 0.008314764498569193, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006775626666666668, "avg_duration_per_task_ms": 60620.74, "avg_output_tokens_per_task": 2327.0066666666667, "last_run_timestamp": "2026-10-01T19:04:36.194Z" } ] }