{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-10-25", "retrieved_at": "2026-10-25T01:45:32.918Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2274, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.964646, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09468125757575757, "avg_duration_per_task_ms": 101980.28282828283, "avg_output_tokens_per_task": 8439.414141414141, "last_run_timestamp": "2026-10-14T15:00:03.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon", "model_slug": "google/gemini-4-argon", "model_name": "Gemini 4 Argon", "provider_name": "auto-routing", "accuracy": 0.959596, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.029260161616161615, "avg_duration_per_task_ms": 40252.82323232323, "avg_output_tokens_per_task": 2870.0252525252527, "last_run_timestamp": "2026-10-23T07:38:37.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9483723333333332, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20306945454545455, "avg_duration_per_task_ms": 134835.03535353535, "avg_output_tokens_per_task": 16798.409090909092, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.9482614885496183, "accuracy_std_dev": 0.0012429999999999941, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07536056743002545, "avg_duration_per_task_ms": 132814.94147582698, "avg_output_tokens_per_task": 18975.51145038168, "last_run_timestamp": "2026-10-15T21:00:04.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.947917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 96, "avg_cost_per_task_usd": 0.009331666666666667, "avg_duration_per_task_ms": 232878.71875, "avg_output_tokens_per_task": 10278.75, "last_run_timestamp": "2026-10-14T18:00:04.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021931015151515153, "avg_duration_per_task_ms": 68600.9191919192, "avg_output_tokens_per_task": 1835.4242424242425, "last_run_timestamp": "2026-10-03T01:13:05.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11987038888888889, "avg_duration_per_task_ms": 35572.169191919194, "avg_output_tokens_per_task": 2020.9419191919192, "last_run_timestamp": "2026-10-01T08:24:02.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.9393935, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.29793838636363634, "avg_duration_per_task_ms": 106861.7196969697, "avg_output_tokens_per_task": 11692.656565656565, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9391951653543308, "accuracy_std_dev": 0.009884880879403648, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.026313035433070862, "avg_duration_per_task_ms": 56131.107611548556, "avg_output_tokens_per_task": 7729.320209973754, "last_run_timestamp": "2026-10-15T21:01:11.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9371493333333333, "accuracy_std_dev": 0.004199386807089248, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.30925494444444446, "avg_duration_per_task_ms": 152328.37037037036, "avg_output_tokens_per_task": 10179.092592592593, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.9353121872146118, "accuracy_std_dev": 0.0026674999999999893, "run_count": 2, "total_questions": 219, "avg_cost_per_task_usd": 0.014858730593607304, "avg_duration_per_task_ms": 395134.2511415525, "avg_output_tokens_per_task": 29583.762557077625, "last_run_timestamp": "2026-10-24T08:19:41.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash", "model_slug": "google/gemini-3.8-flash", "model_name": "Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.9343430000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023175040404040403, "avg_duration_per_task_ms": 44381.02525252525, "avg_output_tokens_per_task": 6131.126262626262, "last_run_timestamp": "2026-10-23T07:37:43.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.9337823333333333, "accuracy_std_dev": 0.008398773614178557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3842955404040404, "avg_duration_per_task_ms": 82918.1936026936, "avg_output_tokens_per_task": 11996.59090909091, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9332206666666667, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.24476150168350166, "avg_duration_per_task_ms": 101605.01851851853, "avg_output_tokens_per_task": 11508.424242424242, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03490100505050505, "avg_duration_per_task_ms": 57907.71885521885, "avg_output_tokens_per_task": 6070.841750841751, "last_run_timestamp": "2026-10-09T23:23:36.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9326596250000001, "accuracy_std_dev": 0.00862529761714779, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.32610833964646463, "avg_duration_per_task_ms": 161836.85416666666, "avg_output_tokens_per_task": 10112.390782828283, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9325395714285714, "accuracy_std_dev": 0.004401689845956867, "run_count": 4, "total_questions": 756, "avg_cost_per_task_usd": 0.06429072222222224, "avg_duration_per_task_ms": 63071.10317460317, "avg_output_tokens_per_task": 11529.871693121693, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9308640666666669, "accuracy_std_dev": 0.007021551143602255, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.026242806060606066, "avg_duration_per_task_ms": 18771.915488215487, "avg_output_tokens_per_task": 2231.500336700337, "last_run_timestamp": "2026-10-24T08:30:19.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.9306396, "accuracy_std_dev": 0.006683344480123684, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025279440404040406, "avg_duration_per_task_ms": 20334.968686868688, "avg_output_tokens_per_task": 2296.2020202020203, "last_run_timestamp": "2026-10-24T08:11:43.558Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.930415, "accuracy_std_dev": 0.004827415699246697, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06216489141414142, "avg_duration_per_task_ms": 64761.851851851854, "avg_output_tokens_per_task": 11730.429292929293, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.9303028, "accuracy_std_dev": 0.00377921650081072, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02488049898989899, "avg_duration_per_task_ms": 19525.145454545454, "avg_output_tokens_per_task": 2247.960606060606, "last_run_timestamp": "2026-10-24T08:10:23.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9280388242811503, "accuracy_std_dev": 0.010739874350082303, "run_count": 12, "total_questions": 2191, "avg_cost_per_task_usd": 0.1293467288909174, "avg_duration_per_task_ms": 132537.31081697854, "avg_output_tokens_per_task": 8542.681424007302, "last_run_timestamp": "2026-10-24T08:42:17.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9276092, "accuracy_std_dev": 0.002380828544855767, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025585643434343434, "avg_duration_per_task_ms": 19359.4303030303, "avg_output_tokens_per_task": 2318.8838383838383, "last_run_timestamp": "2026-10-24T08:01:41.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.9271882499999999, "accuracy_std_dev": 0.006560703749408305, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025096623737373737, "avg_duration_per_task_ms": 19787.91414141414, "avg_output_tokens_per_task": 2272.02904040404, "last_run_timestamp": "2026-10-24T08:11:40.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9266735746971735, "accuracy_std_dev": 0.007620373920907321, "run_count": 36, "total_questions": 6687, "avg_cost_per_task_usd": 0.16851088829071328, "avg_duration_per_task_ms": 159036.4706146254, "avg_output_tokens_per_task": 9187.432929564828, "last_run_timestamp": "2026-10-24T08:50:28.834Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.004418715424192887, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05958464141414141, "avg_duration_per_task_ms": 58983.43771043771, "avg_output_tokens_per_task": 11740.584175084176, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9259260000000001, "accuracy_std_dev": 0.004610436378478734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10788488484848485, "avg_duration_per_task_ms": 187765.80707070706, "avg_output_tokens_per_task": 9499.632323232323, "last_run_timestamp": "2026-10-24T08:37:09.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.925926, "accuracy_std_dev": 0.005498287975967322, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1441699393939394, "avg_duration_per_task_ms": 61658.986531986535, "avg_output_tokens_per_task": 15894.8367003367, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9259256666666668, "accuracy_std_dev": 0.0036366213562713575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1413088872053872, "avg_duration_per_task_ms": 82468.60606060606, "avg_output_tokens_per_task": 15584.752525252525, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.925795996178344, "accuracy_std_dev": 0.007145577022492671, "run_count": 8, "total_questions": 1570, "avg_cost_per_task_usd": 0.1095707592356688, "avg_duration_per_task_ms": 200872.2719745223, "avg_output_tokens_per_task": 9447.89617834395, "last_run_timestamp": "2026-10-24T08:14:36.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9255440309278352, "accuracy_std_dev": 0.0047577793839843644, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.3407643092783505, "avg_duration_per_task_ms": 132309.46391752578, "avg_output_tokens_per_task": 10197.671821305841, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9248735, "accuracy_std_dev": 0.00862277742377712, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10364464204545455, "avg_duration_per_task_ms": 55123.33648989899, "avg_output_tokens_per_task": 3517.8642676767677, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.9247310806451613, "accuracy_std_dev": 0.013119500000000006, "run_count": 2, "total_questions": 372, "avg_cost_per_task_usd": 0.005701161290322581, "avg_duration_per_task_ms": 222245.45967741936, "avg_output_tokens_per_task": 19159.182795698925, "last_run_timestamp": "2026-10-24T08:13:25.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9242421666666667, "accuracy_std_dev": 0.0048599613653014136, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.14878865656565657, "avg_duration_per_task_ms": 85040.50420875421, "avg_output_tokens_per_task": 15641.649831649831, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto-26.10-preview", "model_slug": "unbiased/pareto-26.10-preview", "model_name": "Pareto 26.10 Preview", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0030723434343434344, "avg_duration_per_task_ms": 37807.181818181816, "avg_output_tokens_per_task": 903.9090909090909, "last_run_timestamp": "2026-10-15T14:24:31.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.008625346413333228, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05533751388888889, "avg_duration_per_task_ms": 61705.84722222222, "avg_output_tokens_per_task": 3564.286616161616, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9230042575935954, "accuracy_std_dev": 0.028834365810508387, "run_count": 22, "total_questions": 4247, "avg_cost_per_task_usd": 0.12647735083588418, "avg_duration_per_task_ms": 173368.8584883447, "avg_output_tokens_per_task": 9831.030138921591, "last_run_timestamp": "2026-10-24T08:45:15.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9227054347826086, "accuracy_std_dev": 0.0037342422507786015, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.09927257048748353, "avg_duration_per_task_ms": 52555.73781291173, "avg_output_tokens_per_task": 3495.6416337285905, "last_run_timestamp": "2026-10-01T19:12:57.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9220161548643283, "accuracy_std_dev": 0.015489389982329555, "run_count": 9, "total_questions": 1511, "avg_cost_per_task_usd": 0.1115360423560556, "avg_duration_per_task_ms": 205913.44606221045, "avg_output_tokens_per_task": 8295.92322964924, "last_run_timestamp": "2026-10-24T08:11:11.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9214363999999999, "accuracy_std_dev": 0.007595474019001951, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.13534853400673405, "avg_duration_per_task_ms": 180051.0377104377, "avg_output_tokens_per_task": 11618.102693602694, "last_run_timestamp": "2026-10-24T09:13:18.915Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9214090472319009, "accuracy_std_dev": 0.0068651349363197224, "run_count": 14, "total_questions": 2583, "avg_cost_per_task_usd": 0.14661487727448702, "avg_duration_per_task_ms": 125761.86372435153, "avg_output_tokens_per_task": 9628.765001935733, "last_run_timestamp": "2026-10-24T08:48:19.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.92087525, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.14864516792929292, "avg_duration_per_task_ms": 124980.99747474748, "avg_output_tokens_per_task": 9671.253787878788, "last_run_timestamp": "2026-10-24T08:58:50.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06043574579124578, "avg_duration_per_task_ms": 62971.38047138047, "avg_output_tokens_per_task": 3586.8703703703704, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.920127, "accuracy_std_dev": 0.007058817385999527, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.16989762008978673, "avg_duration_per_task_ms": 298475.25589225587, "avg_output_tokens_per_task": 9659.37598204265, "last_run_timestamp": "2026-10-24T08:46:01.110Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.9200335, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021749656565656566, "avg_duration_per_task_ms": 75214.94191919192, "avg_output_tokens_per_task": 20570.64898989899, "last_run_timestamp": "2026-10-24T08:29:16.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.9196215425531915, "accuracy_std_dev": 0.0047267721426877685, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.14787351063829787, "avg_duration_per_task_ms": 388234.25177304965, "avg_output_tokens_per_task": 9645.446808510638, "last_run_timestamp": "2026-10-24T08:25:42.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9194288167597765, "accuracy_std_dev": 0.012289026624910344, "run_count": 14, "total_questions": 2685, "avg_cost_per_task_usd": 0.12730921601489759, "avg_duration_per_task_ms": 123644.88379888269, "avg_output_tokens_per_task": 8486.700558659219, "last_run_timestamp": "2026-10-24T08:44:47.181Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9194090050632909, "accuracy_std_dev": 0.01100095968888121, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.1314646818565401, "avg_duration_per_task_ms": 272126.83670886076, "avg_output_tokens_per_task": 10234.781012658228, "last_run_timestamp": "2026-10-24T08:16:43.073Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026391550505050505, "avg_duration_per_task_ms": 26392.646464646463, "avg_output_tokens_per_task": 2240.712121212121, "last_run_timestamp": "2026-10-01T08:37:17.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9180774433042584, "accuracy_std_dev": 0.01435885589070384, "run_count": 10, "total_questions": 1949, "avg_cost_per_task_usd": 0.24670780708055412, "avg_duration_per_task_ms": 414365.66136480245, "avg_output_tokens_per_task": 36895.286813750645, "last_run_timestamp": "2026-10-24T08:18:32.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9172978750000002, "accuracy_std_dev": 0.007642562159339972, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09133790782828283, "avg_duration_per_task_ms": 290431.10353535356, "avg_output_tokens_per_task": 24957.965909090908, "last_run_timestamp": "2026-10-24T09:01:06.737Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9171716, "accuracy_std_dev": 0.0037495390703391955, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.02583652727272727, "avg_duration_per_task_ms": 19190.16717171717, "avg_output_tokens_per_task": 2225.3469696969696, "last_run_timestamp": "2026-10-24T08:28:10.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9165345411718132, "accuracy_std_dev": 0.00728827618680496, "run_count": 13, "total_questions": 2526, "avg_cost_per_task_usd": 0.14153307442596988, "avg_duration_per_task_ms": 283509.3226444972, "avg_output_tokens_per_task": 9267.701108471892, "last_run_timestamp": "2026-10-24T08:44:42.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.9158245, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026372479797979794, "avg_duration_per_task_ms": 193752.02525252526, "avg_output_tokens_per_task": 21866.449494949495, "last_run_timestamp": "2026-10-24T08:46:05.254Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9152636666666667, "accuracy_std_dev": 0.006963814615568124, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09072644781144781, "avg_duration_per_task_ms": 657647.413580247, "avg_output_tokens_per_task": 40463.242985409655, "last_run_timestamp": "2026-10-24T08:13:28.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.9147832881355932, "accuracy_std_dev": 0.01218950000000002, "run_count": 2, "total_questions": 354, "avg_cost_per_task_usd": 0.02172540395480226, "avg_duration_per_task_ms": 124806.83333333333, "avg_output_tokens_per_task": 19165.833333333332, "last_run_timestamp": "2026-10-24T08:44:43.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.9142155073529412, "accuracy_std_dev": 0.012753000000000014, "run_count": 2, "total_questions": 272, "avg_cost_per_task_usd": 0.014779014705882354, "avg_duration_per_task_ms": 301053.6323529412, "avg_output_tokens_per_task": 14660.216911764706, "last_run_timestamp": "2026-10-24T08:32:55.335Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9138823076923077, "accuracy_std_dev": 0.013004232597265677, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03398491258741259, "avg_duration_per_task_ms": 227434.108003108, "avg_output_tokens_per_task": 28148.493395493395, "last_run_timestamp": "2026-10-24T16:21:24.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9138354912891987, "accuracy_std_dev": 0.029291397755522402, "run_count": 24, "total_questions": 4592, "avg_cost_per_task_usd": 0.14595076415505223, "avg_duration_per_task_ms": 388256.5635888502, "avg_output_tokens_per_task": 9056.975174216028, "last_run_timestamp": "2026-10-24T08:17:56.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.9132995, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.022403944444444445, "avg_duration_per_task_ms": 147814.9898989899, "avg_output_tokens_per_task": 23595.60606060606, "last_run_timestamp": "2026-10-24T08:18:32.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9131792857142857, "accuracy_std_dev": 0.008577350568532835, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15955438744588746, "avg_duration_per_task_ms": 565778.9992784993, "avg_output_tokens_per_task": 39975.08946608946, "last_run_timestamp": "2026-10-24T08:28:29.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9131455263157894, "accuracy_std_dev": 0.01187053215487832, "run_count": 10, "total_questions": 1349, "avg_cost_per_task_usd": 0.11814906300963676, "avg_duration_per_task_ms": 186376.30022238696, "avg_output_tokens_per_task": 8107.576723498888, "last_run_timestamp": "2026-10-24T08:02:27.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9118396713596139, "accuracy_std_dev": 0.015391401769750214, "run_count": 13, "total_questions": 2486, "avg_cost_per_task_usd": 0.1705156729686243, "avg_duration_per_task_ms": 504972.3576025744, "avg_output_tokens_per_task": 11645.380128720837, "last_run_timestamp": "2026-10-24T08:25:10.474Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9117844, "accuracy_std_dev": 0.006079420781620585, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03790721616161616, "avg_duration_per_task_ms": 129456.97070707071, "avg_output_tokens_per_task": 28484.87575757576, "last_run_timestamp": "2026-10-03T09:28:48.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.911616, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007887881313131314, "avg_duration_per_task_ms": 185118.85353535353, "avg_output_tokens_per_task": 21770.37626262626, "last_run_timestamp": "2026-10-24T08:29:45.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.911111096, "accuracy_std_dev": 0.009498497417355022, "run_count": 4, "total_questions": 750, "avg_cost_per_task_usd": 0.136844476, "avg_duration_per_task_ms": 321154.29333333333, "avg_output_tokens_per_task": 35843.412, "last_run_timestamp": "2026-10-24T08:11:11.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.9108864666666666, "accuracy_std_dev": 0.015101135925780192, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.13889531952861953, "avg_duration_per_task_ms": 583838.6407407407, "avg_output_tokens_per_task": 39957.99865319865, "last_run_timestamp": "2026-10-24T08:16:23.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.9099324999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02721213888888889, "avg_duration_per_task_ms": 211865.40151515152, "avg_output_tokens_per_task": 21376.03787878788, "last_run_timestamp": "2026-10-24T08:25:08.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.9099324999999999, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02111154797979798, "avg_duration_per_task_ms": 145444.96212121213, "avg_output_tokens_per_task": 21297.967171717173, "last_run_timestamp": "2026-10-24T08:04:44.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.909652, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025303424242424242, "avg_duration_per_task_ms": 91659.5925925926, "avg_output_tokens_per_task": 20911.968013468013, "last_run_timestamp": "2026-10-24T08:24:39.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9094274, "accuracy_std_dev": 0.002474202384608023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07117444141414142, "avg_duration_per_task_ms": 135218.17474747475, "avg_output_tokens_per_task": 27695.128282828282, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0054125, "avg_duration_per_task_ms": 235839.91414141413, "avg_output_tokens_per_task": 10749.717171717171, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.908733465037911, "accuracy_std_dev": 0.006850022329140406, "run_count": 12, "total_questions": 2374, "avg_cost_per_task_usd": 0.030451491996630154, "avg_duration_per_task_ms": 408677.97219882056, "avg_output_tokens_per_task": 31497.24094355518, "last_run_timestamp": "2026-10-24T16:30:24.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9084910475789473, "accuracy_std_dev": 0.04945617807940908, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.14244542484210523, "avg_duration_per_task_ms": 331138.315368421, "avg_output_tokens_per_task": 10400.368421052632, "last_run_timestamp": "2026-10-24T08:48:54.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.9082801312101909, "accuracy_std_dev": 0.007583269228208922, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.021053796178343947, "avg_duration_per_task_ms": 83570.12993630573, "avg_output_tokens_per_task": 20735.173248407642, "last_run_timestamp": "2026-10-24T08:31:24.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9080194545454545, "accuracy_std_dev": 0.015874287061234574, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.035897662534435264, "avg_duration_per_task_ms": 311230.74242424243, "avg_output_tokens_per_task": 29833.370982552802, "last_run_timestamp": "2026-10-24T16:24:24.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.9077097088435375, "accuracy_std_dev": 0.01602150486845725, "run_count": 4, "total_questions": 735, "avg_cost_per_task_usd": 0.12614392789115647, "avg_duration_per_task_ms": 1486736.511564626, "avg_output_tokens_per_task": 50215.49115646259, "last_run_timestamp": "2026-10-24T08:43:13.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.9074072500000001, "accuracy_std_dev": 0.008828586845441347, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.009289530303030303, "avg_duration_per_task_ms": 172567.80050505052, "avg_output_tokens_per_task": 18481.776515151516, "last_run_timestamp": "2026-10-24T08:44:46.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.907407, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018446676767676768, "avg_duration_per_task_ms": 140631.38888888888, "avg_output_tokens_per_task": 20189.31818181818, "last_run_timestamp": "2026-10-24T08:54:33.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.907407, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.012070199494949497, "avg_duration_per_task_ms": 98657.74747474748, "avg_output_tokens_per_task": 21841.54797979798, "last_run_timestamp": "2026-10-24T08:21:56.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.907407, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010421979797979798, "avg_duration_per_task_ms": 107076.06565656565, "avg_output_tokens_per_task": 18400.97474747475, "last_run_timestamp": "2026-10-24T08:14:51.110Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9071381724573594, "accuracy_std_dev": 0.03371213456235009, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.015826173089071385, "avg_duration_per_task_ms": 64047.12571067593, "avg_output_tokens_per_task": 3604.423246999368, "last_run_timestamp": "2026-09-30T03:11:21.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.906910242707118, "accuracy_std_dev": 0.038342984458019556, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.14211418241929213, "avg_duration_per_task_ms": 216494.0859587709, "avg_output_tokens_per_task": 9532.819914430183, "last_run_timestamp": "2026-10-24T08:46:33.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9067057499999999, "accuracy_std_dev": 0.02181467158101399, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.15476828703703704, "avg_duration_per_task_ms": 601020.0946969697, "avg_output_tokens_per_task": 39038.01388888889, "last_run_timestamp": "2026-10-24T08:50:20.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9066696814159291, "accuracy_std_dev": 0.02221528527057198, "run_count": 11, "total_questions": 2034, "avg_cost_per_task_usd": 0.150266023107178, "avg_duration_per_task_ms": 558942.4459193707, "avg_output_tokens_per_task": 37986.727138643066, "last_run_timestamp": "2026-10-24T09:27:08.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.9065654999999999, "accuracy_std_dev": 0.008117672649842422, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0029748573232323233, "avg_duration_per_task_ms": 218585.40025252526, "avg_output_tokens_per_task": 19647.71212121212, "last_run_timestamp": "2026-10-24T08:58:07.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9065625122807017, "accuracy_std_dev": 0.025017422490793098, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.035798040545808964, "avg_duration_per_task_ms": 238869.84015594542, "avg_output_tokens_per_task": 30114.881091617935, "last_run_timestamp": "2026-10-24T16:31:25.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9064251666666665, "accuracy_std_dev": 0.020964442642377977, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03777529545454546, "avg_duration_per_task_ms": 260085.33417508416, "avg_output_tokens_per_task": 31644.78872053872, "last_run_timestamp": "2026-10-24T16:42:35.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.9063117514792899, "accuracy_std_dev": 0.0007095000000000296, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.0210026775147929, "avg_duration_per_task_ms": 63992.53550295858, "avg_output_tokens_per_task": 17351.381656804733, "last_run_timestamp": "2026-10-24T08:13:28.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9060604000000001, "accuracy_std_dev": 0.0032644069966840917, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02883772424242424, "avg_duration_per_task_ms": 166818.5292929293, "avg_output_tokens_per_task": 28846.466666666667, "last_run_timestamp": "2026-10-24T08:33:28.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.9059828461538462, "accuracy_std_dev": 0.02626261808627545, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.04094654972804972, "avg_duration_per_task_ms": 283710.56487956486, "avg_output_tokens_per_task": 31884.02874902875, "last_run_timestamp": "2026-10-24T16:17:39.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9058946753684209, "accuracy_std_dev": 0.029872349099497063, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.10017258273684211, "avg_duration_per_task_ms": 566875.4896842105, "avg_output_tokens_per_task": 39376.88463157895, "last_run_timestamp": "2026-10-24T08:54:07.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.905724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06900820707070707, "avg_duration_per_task_ms": 101485.72222222222, "avg_output_tokens_per_task": 16020.974747474747, "last_run_timestamp": "2026-10-24T08:27:04.527Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9057143142857144, "accuracy_std_dev": 0.001963571554591265, "run_count": 4, "total_questions": 700, "avg_cost_per_task_usd": 0.18660921714285714, "avg_duration_per_task_ms": 435582.91714285716, "avg_output_tokens_per_task": 41735.86142857143, "last_run_timestamp": "2026-10-24T08:46:32.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.9056120000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.06647105102040816, "avg_duration_per_task_ms": 64791.01530612245, "avg_output_tokens_per_task": 15327.612244897959, "last_run_timestamp": "2026-10-24T08:14:36.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.9053030000000001, "accuracy_std_dev": 0.004187691070267732, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026182512626262627, "avg_duration_per_task_ms": 138194.00252525252, "avg_output_tokens_per_task": 21714.06186868687, "last_run_timestamp": "2026-10-24T08:02:23.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9052427142857143, "accuracy_std_dev": 0.007830218017659632, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03489360533910534, "avg_duration_per_task_ms": 228842.85353535353, "avg_output_tokens_per_task": 28483.24531024531, "last_run_timestamp": "2026-10-24T08:15:15.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.9051626666666666, "accuracy_std_dev": 0.0055553002519115975, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009504934343434343, "avg_duration_per_task_ms": 118639.06902356903, "avg_output_tokens_per_task": 16467.956228956227, "last_run_timestamp": "2026-10-24T08:45:01.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.904882, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.022279229797979798, "avg_duration_per_task_ms": 122775.77777777778, "avg_output_tokens_per_task": 22121.87626262626, "last_run_timestamp": "2026-10-24T08:33:42.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9047062578763128, "accuracy_std_dev": 0.0178723281096552, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.03395674834694671, "avg_duration_per_task_ms": 313164.32672112016, "avg_output_tokens_per_task": 29780.11707506807, "last_run_timestamp": "2026-10-24T16:12:49.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.9044775343283582, "accuracy_std_dev": 0.043758500000000033, "run_count": 2, "total_questions": 335, "avg_cost_per_task_usd": 0.01697948358208955, "avg_duration_per_task_ms": 145286.75820895523, "avg_output_tokens_per_task": 24074.814925373135, "last_run_timestamp": "2026-10-24T08:44:22.197Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9044612500000001, "accuracy_std_dev": 0.009172855468037184, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03501683964646465, "avg_duration_per_task_ms": 199099.8143939394, "avg_output_tokens_per_task": 29016.732954545456, "last_run_timestamp": "2026-10-24T08:11:11.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.904461125, "accuracy_std_dev": 0.008737580020198659, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02118971275252525, "avg_duration_per_task_ms": 207950.51199494948, "avg_output_tokens_per_task": 28934.199494949495, "last_run_timestamp": "2026-10-24T08:11:11.332Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9040400000000001, "accuracy_std_dev": 0.004761657064510192, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03280091414141414, "avg_duration_per_task_ms": 814621.6686868687, "avg_output_tokens_per_task": 37346.743434343436, "last_run_timestamp": "2026-10-24T08:23:31.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.9039929892328398, "accuracy_std_dev": 0.013615822071674532, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.061841156123822336, "avg_duration_per_task_ms": 74303.45625841184, "avg_output_tokens_per_task": 14413.555854643339, "last_run_timestamp": "2026-10-24T08:25:42.800Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9036160645842126, "accuracy_std_dev": 0.011349068793175144, "run_count": 12, "total_questions": 2369, "avg_cost_per_task_usd": 0.028033349514563106, "avg_duration_per_task_ms": 292905.91599831154, "avg_output_tokens_per_task": 23207.665681722247, "last_run_timestamp": "2026-10-24T16:03:25.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9036070050669563, "accuracy_std_dev": 0.012155177188745377, "run_count": 14, "total_questions": 2763, "avg_cost_per_task_usd": 0.027082988418385807, "avg_duration_per_task_ms": 186339.8273615635, "avg_output_tokens_per_task": 25900.92254795512, "last_run_timestamp": "2026-10-24T08:48:00.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9033163440134907, "accuracy_std_dev": 0.008355089002931883, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.015168129286115795, "avg_duration_per_task_ms": 392884.92523889826, "avg_output_tokens_per_task": 28918.40921866217, "last_run_timestamp": "2026-10-24T08:18:49.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.9031985, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008858196969696969, "avg_duration_per_task_ms": 108844.08838383839, "avg_output_tokens_per_task": 17385.492424242424, "last_run_timestamp": "2026-10-24T09:00:40.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.9031626720031235, "accuracy_std_dev": 0.019975453712141735, "run_count": 13, "total_questions": 2561, "avg_cost_per_task_usd": 0.11988065286997265, "avg_duration_per_task_ms": 343221.44474814524, "avg_output_tokens_per_task": 8205.648574775478, "last_run_timestamp": "2026-10-24T08:49:03.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902918, "accuracy_std_dev": 0.0007933738084913016, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14672305387205387, "avg_duration_per_task_ms": 133620.78956228957, "avg_output_tokens_per_task": 9652.131313131313, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9026110979153505, "accuracy_std_dev": 0.006660054560587301, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.028997156664560955, "avg_duration_per_task_ms": 168876.1749842072, "avg_output_tokens_per_task": 28895.867340492736, "last_run_timestamp": "2026-10-24T08:51:57.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.9023569999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06824257323232323, "avg_duration_per_task_ms": 275518.9696969697, "avg_output_tokens_per_task": 15830.017676767677, "last_run_timestamp": "2026-10-24T08:04:43.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.9023568, "accuracy_std_dev": 0.005733794394639539, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.05410575151515151, "avg_duration_per_task_ms": 31371.09595959596, "avg_output_tokens_per_task": 2496.787878787879, "last_run_timestamp": "2026-10-24T08:04:44.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9023566666666667, "accuracy_std_dev": 0.003636621356271357, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.053918992424242425, "avg_duration_per_task_ms": 26868.823232323233, "avg_output_tokens_per_task": 2480.0361952861954, "last_run_timestamp": "2026-10-24T08:14:51.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Nebius", "accuracy": 0.9017956666666668, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014097868686868686, "avg_duration_per_task_ms": 101233.50336700337, "avg_output_tokens_per_task": 13920.223905723906, "last_run_timestamp": "2026-10-24T08:29:16.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.9017254583333333, "accuracy_std_dev": 0.0058683477301193655, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.2056731323653199, "avg_duration_per_task_ms": 97891.09090909091, "avg_output_tokens_per_task": 7837.397516835017, "last_run_timestamp": "2026-10-24T08:44:41.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.9016089931034483, "accuracy_std_dev": 0.017035826057784066, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.10899811586206896, "avg_duration_per_task_ms": 561470.3995402299, "avg_output_tokens_per_task": 37112.96183908046, "last_run_timestamp": "2026-10-24T08:01:41.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-1017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "Deepseek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.901515, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006598151515151515, "avg_duration_per_task_ms": 72990.44444444444, "avg_output_tokens_per_task": 11623.267676767677, "last_run_timestamp": "2026-10-23T06:09:54.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.022678441919191918, "avg_duration_per_task_ms": 49322.41161616162, "avg_output_tokens_per_task": 18730.81818181818, "last_run_timestamp": "2026-10-24T08:41:23.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.9009352959183673, "accuracy_std_dev": 0.0002650000000000152, "run_count": 2, "total_questions": 392, "avg_cost_per_task_usd": 0.013316757653061225, "avg_duration_per_task_ms": 199818.51020408163, "avg_output_tokens_per_task": 22103.426020408162, "last_run_timestamp": "2026-10-24T08:16:43.075Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9008097837837836, "accuracy_std_dev": 0.0057041149259447295, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.21664884998635, "avg_duration_per_task_ms": 93919.68495768496, "avg_output_tokens_per_task": 7898.381790881791, "last_run_timestamp": "2026-10-24T08:14:03.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.9006734999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.020602151515151518, "avg_duration_per_task_ms": 398490.0909090909, "avg_output_tokens_per_task": 16966.28787878788, "last_run_timestamp": "2026-10-24T08:44:49.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.9006730000000001, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026410239898989902, "avg_duration_per_task_ms": 92596.52777777778, "avg_output_tokens_per_task": 21830.75, "last_run_timestamp": "2026-10-24T08:32:55.338Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.900673, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07405119191919192, "avg_duration_per_task_ms": 94171.00505050505, "avg_output_tokens_per_task": 17264.035353535353, "last_run_timestamp": "2026-10-24T08:14:51.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9002912013585639, "accuracy_std_dev": 0.008613175564538058, "run_count": 12, "total_questions": 2061, "avg_cost_per_task_usd": 0.024704503153808833, "avg_duration_per_task_ms": 166852.90538573507, "avg_output_tokens_per_task": 20435.77389616691, "last_run_timestamp": "2026-10-24T16:26:43.118Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8999997999999999, "accuracy_std_dev": 0.0027351667883330276, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03378601313131313, "avg_duration_per_task_ms": 836485.5171717171, "avg_output_tokens_per_task": 38467.88484848485, "last_run_timestamp": "2026-10-24T08:37:44.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8999869065420562, "accuracy_std_dev": 0.0055303889812583175, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.012645620327102805, "avg_duration_per_task_ms": 176614.65965732088, "avg_output_tokens_per_task": 26316.880841121496, "last_run_timestamp": "2026-10-24T08:45:10.619Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8999248888888889, "accuracy_std_dev": 0.006354339085213833, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15479229124579125, "avg_duration_per_task_ms": 130090.1857463524, "avg_output_tokens_per_task": 9547.675645342311, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8998315555555554, "accuracy_std_dev": 0.004532988273904737, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.05740673849607183, "avg_duration_per_task_ms": 25170.2278338945, "avg_output_tokens_per_task": 2482.2177328843995, "last_run_timestamp": "2026-10-24T08:41:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.8998315, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02710091414141414, "avg_duration_per_task_ms": 156920.601010101, "avg_output_tokens_per_task": 22433.31818181818, "last_run_timestamp": "2026-10-24T08:34:01.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.8998315, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11557131818181818, "avg_duration_per_task_ms": 1030735.8762626263, "avg_output_tokens_per_task": 45994.38383838384, "last_run_timestamp": "2026-10-24T08:14:03.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14109027272727273, "avg_duration_per_task_ms": 131303, "avg_output_tokens_per_task": 9278.422558922559, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.899551, "accuracy_std_dev": 0.006198077174952426, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0412280202020202, "avg_duration_per_task_ms": 141835.40572390571, "avg_output_tokens_per_task": 27211.210437710437, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.8995506666666668, "accuracy_std_dev": 0.005722799567422267, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11900113804713805, "avg_duration_per_task_ms": 846691.8215488215, "avg_output_tokens_per_task": 47399.26936026936, "last_run_timestamp": "2026-10-24T08:02:41.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8993638888888889, "accuracy_std_dev": 0.004257353623619148, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.058306676767676764, "avg_duration_per_task_ms": 37477.73063973064, "avg_output_tokens_per_task": 2520.0830527497196, "last_run_timestamp": "2026-10-24T08:44:59.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8992834083438684, "accuracy_std_dev": 0.01959300954191881, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.12703601896333752, "avg_duration_per_task_ms": 516266.79730299197, "avg_output_tokens_per_task": 38334.854193004634, "last_run_timestamp": "2026-10-24T08:37:37.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.89913, "accuracy_std_dev": 0.005890503218458193, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05769051388888888, "avg_duration_per_task_ms": 26119.287037037036, "avg_output_tokens_per_task": 2529.9574915824915, "last_run_timestamp": "2026-10-24T08:24:00.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8991226842105264, "accuracy_std_dev": 0.006159935919460174, "run_count": 38, "total_questions": 7524, "avg_cost_per_task_usd": 0.2224983919457735, "avg_duration_per_task_ms": 94191.69205209994, "avg_output_tokens_per_task": 8105.584795321637, "last_run_timestamp": "2026-10-24T08:20:25.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8989900000000001, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007148643939393939, "avg_duration_per_task_ms": 224904.53787878787, "avg_output_tokens_per_task": 25295.88888888889, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.8989899999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07316585858585858, "avg_duration_per_task_ms": 97804.9696969697, "avg_output_tokens_per_task": 16993.328282828283, "last_run_timestamp": "2026-10-24T08:12:37.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.005188652155184398, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03249949747474747, "avg_duration_per_task_ms": 137621.66666666666, "avg_output_tokens_per_task": 26156.715909090908, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8987651999999998, "accuracy_std_dev": 0.01944778431321504, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.08230536969696971, "avg_duration_per_task_ms": 213609.22895622897, "avg_output_tokens_per_task": 30143.43771043771, "last_run_timestamp": "2026-10-24T16:37:39.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.8987310178571428, "accuracy_std_dev": 0.016411985541715763, "run_count": 11, "total_questions": 2128, "avg_cost_per_task_usd": 0.015117266917293233, "avg_duration_per_task_ms": 247345.46381578947, "avg_output_tokens_per_task": 19483.954417293233, "last_run_timestamp": "2026-10-24T08:55:07.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna", "model_slug": "openai/gpt-6-luna", "model_name": "GPT-6-Luna", "provider_name": "auto-routing", "accuracy": 0.898477, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.003442197969543147, "avg_duration_per_task_ms": 73312.30456852792, "avg_output_tokens_per_task": 6831.9390862944165, "last_run_timestamp": "2026-10-08T13:29:57.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8983774545454545, "accuracy_std_dev": 0.004829955304962313, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.2299064136822773, "avg_duration_per_task_ms": 110399.28466483012, "avg_output_tokens_per_task": 8983.414600550965, "last_run_timestamp": "2026-10-24T08:27:26.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.8983162, "accuracy_std_dev": 0.008080975000580047, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1253350707070707, "avg_duration_per_task_ms": 994044.4858585858, "avg_output_tokens_per_task": 49876.97171717172, "last_run_timestamp": "2026-10-24T08:12:02.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8981654060721064, "accuracy_std_dev": 0.006690047440041041, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.01891549778621126, "avg_duration_per_task_ms": 414943.30676786846, "avg_output_tokens_per_task": 22047.87349778621, "last_run_timestamp": "2026-10-24T09:25:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.898164278659612, "accuracy_std_dev": 0.007879477322215058, "run_count": 26, "total_questions": 5103, "avg_cost_per_task_usd": 0.12592846874387617, "avg_duration_per_task_ms": 58143.77189888301, "avg_output_tokens_per_task": 4709.22437781697, "last_run_timestamp": "2026-10-24T08:33:28.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8977654545454545, "accuracy_std_dev": 0.00783374636212555, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.19846787741046834, "avg_duration_per_task_ms": 95325.20982552801, "avg_output_tokens_per_task": 7728.792011019284, "last_run_timestamp": "2026-10-24T08:17:56.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8972558314475225, "accuracy_std_dev": 0.033281996803894424, "run_count": 13, "total_questions": 2563, "avg_cost_per_task_usd": 0.1455034553257901, "avg_duration_per_task_ms": 811067.1357783847, "avg_output_tokens_per_task": 36442.67655091689, "last_run_timestamp": "2026-10-24T08:25:27.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8972168045540795, "accuracy_std_dev": 0.016285736546419217, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.024440066413662233, "avg_duration_per_task_ms": 243796.87760910817, "avg_output_tokens_per_task": 26883.111321948134, "last_run_timestamp": "2026-10-24T08:56:13.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8971659166666668, "accuracy_std_dev": 0.011261007344359652, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19972315109427613, "avg_duration_per_task_ms": 95894.5180976431, "avg_output_tokens_per_task": 7770.697390572391, "last_run_timestamp": "2026-10-24T08:28:56.790Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8968568964143425, "accuracy_std_dev": 0.005278733293840445, "run_count": 4, "total_questions": 753, "avg_cost_per_task_usd": 0.15754132403718457, "avg_duration_per_task_ms": 84099.57768924303, "avg_output_tokens_per_task": 9420.674634794157, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "CoreWeave", "accuracy": 0.8964644999999999, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.027913444444444446, "avg_duration_per_task_ms": 54299.99747474748, "avg_output_tokens_per_task": 13871.598484848484, "last_run_timestamp": "2026-10-24T08:12:49.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8962877525832377, "accuracy_std_dev": 0.04356588502231736, "run_count": 9, "total_questions": 1742, "avg_cost_per_task_usd": 0.04917616016073478, "avg_duration_per_task_ms": 485084.33180252585, "avg_output_tokens_per_task": 15776.056831228472, "last_run_timestamp": "2026-10-24T16:26:47.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.8962022556962025, "accuracy_std_dev": 0.0022689999999999655, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.020123096202531644, "avg_duration_per_task_ms": 653583.2962025317, "avg_output_tokens_per_task": 42659.516455696205, "last_run_timestamp": "2026-10-10T00:00:04.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8961839333333333, "accuracy_std_dev": 0.006639909331375588, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.2064141511784512, "avg_duration_per_task_ms": 100220.49057239057, "avg_output_tokens_per_task": 8041.884511784512, "last_run_timestamp": "2026-10-24T08:30:35.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.8960434999999999, "accuracy_std_dev": 0.007996499999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06795408585858587, "avg_duration_per_task_ms": 278542.00757575757, "avg_output_tokens_per_task": 15808.588383838383, "last_run_timestamp": "2026-10-24T08:15:15.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8959899189640768, "accuracy_std_dev": 0.02896400676270464, "run_count": 8, "total_questions": 1197, "avg_cost_per_task_usd": 0.024112938178780283, "avg_duration_per_task_ms": 783093.3592314118, "avg_output_tokens_per_task": 22333.22807017544, "last_run_timestamp": "2026-10-17T16:03:42.308Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.895623, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014240813131313132, "avg_duration_per_task_ms": 199874.40656565657, "avg_output_tokens_per_task": 21243.522727272728, "last_run_timestamp": "2026-10-24T08:54:21.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.8956225, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026159166666666667, "avg_duration_per_task_ms": 177167.38888888888, "avg_output_tokens_per_task": 21649.300505050505, "last_run_timestamp": "2026-10-24T08:22:00.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8954932820512822, "accuracy_std_dev": 0.00772304896264559, "run_count": 39, "total_questions": 7722, "avg_cost_per_task_usd": 0.2042425423465424, "avg_duration_per_task_ms": 70332.41427091428, "avg_output_tokens_per_task": 7437.828541828542, "last_run_timestamp": "2026-10-24T08:43:08.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8953421666666667, "accuracy_std_dev": 0.0047035083153133565, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05463887542087542, "avg_duration_per_task_ms": 31241.656565656565, "avg_output_tokens_per_task": 2527.686026936027, "last_run_timestamp": "2026-10-24T08:08:54.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8952379124999998, "accuracy_std_dev": 0.0099951687737931, "run_count": 17, "total_questions": 3360, "avg_cost_per_task_usd": 0.013780392857142856, "avg_duration_per_task_ms": 312284.5139880952, "avg_output_tokens_per_task": 27615.769642857143, "last_run_timestamp": "2026-10-24T08:49:27.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8951046153846154, "accuracy_std_dev": 0.007423472104267645, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.12089025990675989, "avg_duration_per_task_ms": 58355.058275058276, "avg_output_tokens_per_task": 4615.157342657342, "last_run_timestamp": "2026-10-24T08:16:38.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8950615, "accuracy_std_dev": 0.01045325149973282, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008697955387205388, "avg_duration_per_task_ms": 170461.9739057239, "avg_output_tokens_per_task": 30584.97053872054, "last_run_timestamp": "2026-10-24T08:08:54.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8950130708661417, "accuracy_std_dev": 0.00891334037763057, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.048151682414698166, "avg_duration_per_task_ms": 43476.24934383202, "avg_output_tokens_per_task": 3503.036745406824, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "BaseTen", "accuracy": 0.8947810000000002, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03846228535353535, "avg_duration_per_task_ms": 31568.290404040403, "avg_output_tokens_per_task": 13846.992424242424, "last_run_timestamp": "2026-10-24T08:14:03.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.894781, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036287184343434344, "avg_duration_per_task_ms": 737979.1590909091, "avg_output_tokens_per_task": 37423.542929292926, "last_run_timestamp": "2026-10-24T08:21:38.159Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8946608571428573, "accuracy_std_dev": 0.021330086138923062, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.025403829725829725, "avg_duration_per_task_ms": 208889.86363636365, "avg_output_tokens_per_task": 27754.472582972583, "last_run_timestamp": "2026-10-24T08:01:41.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8945826942242354, "accuracy_std_dev": 0.013340195542461545, "run_count": 18, "total_questions": 3532, "avg_cost_per_task_usd": 0.003962120045300114, "avg_duration_per_task_ms": 321568.4875424689, "avg_output_tokens_per_task": 21723.382785956965, "last_run_timestamp": "2026-10-24T08:41:47.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8941264444444444, "accuracy_std_dev": 0.009636502052221474, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05178651795735129, "avg_duration_per_task_ms": 45445.343434343435, "avg_output_tokens_per_task": 3671.5897867564536, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8939451085180864, "accuracy_std_dev": 0.009458026084140597, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.22913243290548424, "avg_duration_per_task_ms": 109689.0805134189, "avg_output_tokens_per_task": 8941.389342668223, "last_run_timestamp": "2026-10-24T08:44:41.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.89372875, "accuracy_std_dev": 0.019477743681892395, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02587082891414141, "avg_duration_per_task_ms": 157351.7840909091, "avg_output_tokens_per_task": 28618.30744949495, "last_run_timestamp": "2026-10-24T08:44:42.937Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.8934709999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 194, "avg_cost_per_task_usd": 0.11686864948453608, "avg_duration_per_task_ms": 605687.7474226804, "avg_output_tokens_per_task": 46496.701030927834, "last_run_timestamp": "2026-10-13T03:00:07.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8932483811659193, "accuracy_std_dev": 0.017336749323389345, "run_count": 7, "total_questions": 1338, "avg_cost_per_task_usd": 0.03355175710014948, "avg_duration_per_task_ms": 240320.3542600897, "avg_output_tokens_per_task": 32642.40284005979, "last_run_timestamp": "2026-10-24T08:41:23.835Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.893246, "accuracy_std_dev": 0.014262311738201734, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.015338635769459302, "avg_duration_per_task_ms": 208853.27035056447, "avg_output_tokens_per_task": 25149.005347593582, "last_run_timestamp": "2026-10-24T08:42:04.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.8930975, "accuracy_std_dev": 0.009560420192125432, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11553578787878788, "avg_duration_per_task_ms": 625203.1957070708, "avg_output_tokens_per_task": 46042.48863636364, "last_run_timestamp": "2026-10-24T08:01:41.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8928570861678005, "accuracy_std_dev": 0.015340394005370288, "run_count": 5, "total_questions": 882, "avg_cost_per_task_usd": 0.12683532766439912, "avg_duration_per_task_ms": 285532.1031746032, "avg_output_tokens_per_task": 28366.77097505669, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.89267675, "accuracy_std_dev": 0.006979218773437292, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037834488636363635, "avg_duration_per_task_ms": 50710.07575757576, "avg_output_tokens_per_task": 3480.1502525252527, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.892262208728653, "accuracy_std_dev": 0.011301549411889275, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.20793211669829223, "avg_duration_per_task_ms": 100118.31277672359, "avg_output_tokens_per_task": 8089.089500316256, "last_run_timestamp": "2026-10-24T08:05:46.408Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.8922557999999999, "accuracy_std_dev": 0.007299435934371887, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11377392424242423, "avg_duration_per_task_ms": 848578.0848484848, "avg_output_tokens_per_task": 45335.65555555555, "last_run_timestamp": "2026-10-24T08:01:41.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8922132346791637, "accuracy_std_dev": 0.039246562470616454, "run_count": 15, "total_questions": 2774, "avg_cost_per_task_usd": 0.35474507606344624, "avg_duration_per_task_ms": 458796.4985580389, "avg_output_tokens_per_task": 57478.27541456381, "last_run_timestamp": "2026-10-24T16:31:04.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8919751666666667, "accuracy_std_dev": 0.008661714291768222, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024034642255892257, "avg_duration_per_task_ms": 245030.94696969696, "avg_output_tokens_per_task": 30463.32239057239, "last_run_timestamp": "2026-10-24T08:01:41.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8916709261744967, "accuracy_std_dev": 0.019973452544815567, "run_count": 10, "total_questions": 1937, "avg_cost_per_task_usd": 0.13092793236964376, "avg_duration_per_task_ms": 553822.8373773877, "avg_output_tokens_per_task": 42781.386164171396, "last_run_timestamp": "2026-10-24T09:11:22.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8916082307692307, "accuracy_std_dev": 0.008673677213747568, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.24150008061383055, "avg_duration_per_task_ms": 84772.77195027196, "avg_output_tokens_per_task": 8978.732711732711, "last_run_timestamp": "2026-10-24T08:14:36.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.8915753430656934, "accuracy_std_dev": 0.004887565515332395, "run_count": 6, "total_questions": 1096, "avg_cost_per_task_usd": 0.03103391879562044, "avg_duration_per_task_ms": 795798.2390510949, "avg_output_tokens_per_task": 36089.42244525548, "last_run_timestamp": "2026-10-24T08:07:59.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8915342857142857, "accuracy_std_dev": 0.010905197538189691, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011785490620490622, "avg_duration_per_task_ms": 284433.1883116883, "avg_output_tokens_per_task": 21375.483405483406, "last_run_timestamp": "2026-10-24T08:45:15.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8914141, "accuracy_std_dev": 0.016939939187907378, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1456811595959596, "avg_duration_per_task_ms": 619546.0101010101, "avg_output_tokens_per_task": 36709.93131313131, "last_run_timestamp": "2026-10-17T08:18:00.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.891414, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007142083333333334, "avg_duration_per_task_ms": 125847.88383838384, "avg_output_tokens_per_task": 11714.222222222223, "last_run_timestamp": "2026-10-24T08:16:38.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "DigitalOcean", "accuracy": 0.891414, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016427454545454544, "avg_duration_per_task_ms": 96439.17424242424, "avg_output_tokens_per_task": 13618.247474747475, "last_run_timestamp": "2026-10-24T08:14:51.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8914139999999999, "accuracy_std_dev": 0.010479783311214021, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005115402777777778, "avg_duration_per_task_ms": 224009.38825757575, "avg_output_tokens_per_task": 28058.227904040403, "last_run_timestamp": "2026-10-24T16:05:36.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8907934361904761, "accuracy_std_dev": 0.020177999596946557, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.18863606476190478, "avg_duration_per_task_ms": 1541820.2831746032, "avg_output_tokens_per_task": 55365.57968253968, "last_run_timestamp": "2026-10-24T16:20:08.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.8907719999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 177, "avg_cost_per_task_usd": 0.10733528813559323, "avg_duration_per_task_ms": 969537.395480226, "avg_output_tokens_per_task": 42767.57627118644, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8905723333333333, "accuracy_std_dev": 0.021339306148253502, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13216584343434343, "avg_duration_per_task_ms": 251067.54713804714, "avg_output_tokens_per_task": 43934.003367003366, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.005722799567422281, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.19800758417508418, "avg_duration_per_task_ms": 96141.14141414141, "avg_output_tokens_per_task": 7695.558922558923, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8900110000000001, "accuracy_std_dev": 0.0055552329084086844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2164912121212121, "avg_duration_per_task_ms": 104660.01851851853, "avg_output_tokens_per_task": 8434.90404040404, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8898858525165823, "accuracy_std_dev": 0.009105600456959892, "run_count": 39, "total_questions": 7689, "avg_cost_per_task_usd": 0.133101505137209, "avg_duration_per_task_ms": 59235.32214852387, "avg_output_tokens_per_task": 4759.289244375082, "last_run_timestamp": "2026-10-24T08:16:43.073Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8897974953271028, "accuracy_std_dev": 0.009445547545688906, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.20473812188473522, "avg_duration_per_task_ms": 99561.84657320872, "avg_output_tokens_per_task": 7962.808411214954, "last_run_timestamp": "2026-10-24T08:44:29.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.889547806779661, "accuracy_std_dev": 0.017603851112753716, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.03740084406779662, "avg_duration_per_task_ms": 166568.35593220338, "avg_output_tokens_per_task": 31004.76779661017, "last_run_timestamp": "2026-10-24T08:41:52.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8887654163757536, "accuracy_std_dev": 0.008192069777191544, "run_count": 16, "total_questions": 3151, "avg_cost_per_task_usd": 0.01559116534433513, "avg_duration_per_task_ms": 197619.02634084417, "avg_output_tokens_per_task": 26956.708981275784, "last_run_timestamp": "2026-10-24T08:24:10.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8886482857142856, "accuracy_std_dev": 0.013807459233682208, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1021348318903319, "avg_duration_per_task_ms": 444900.012987013, "avg_output_tokens_per_task": 35591.61255411255, "last_run_timestamp": "2026-10-24T08:42:40.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8885464319465846, "accuracy_std_dev": 0.029017071552622252, "run_count": 10, "total_questions": 1947, "avg_cost_per_task_usd": 0.009370765279917823, "avg_duration_per_task_ms": 312934.7123780175, "avg_output_tokens_per_task": 26808.334874165383, "last_run_timestamp": "2026-10-24T08:13:41.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.888467875, "accuracy_std_dev": 0.02891189332021296, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10564124999999999, "avg_duration_per_task_ms": 446348.9172979798, "avg_output_tokens_per_task": 36610.40214646464, "last_run_timestamp": "2026-10-24T16:12:09.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.8880469999999999, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02377390909090909, "avg_duration_per_task_ms": 131245.66919191918, "avg_output_tokens_per_task": 19627.237373737375, "last_run_timestamp": "2026-10-24T08:41:24.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8877663333333334, "accuracy_std_dev": 0.005555502277522277, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03440870875420875, "avg_duration_per_task_ms": 50195.98148148148, "avg_output_tokens_per_task": 3309.2491582491584, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8869462307692308, "accuracy_std_dev": 0.01086717272801863, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.025130035353535356, "avg_duration_per_task_ms": 337121.5742035742, "avg_output_tokens_per_task": 26419.156565656565, "last_run_timestamp": "2026-10-24T08:11:11.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8867541405460061, "accuracy_std_dev": 0.003543823280018365, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.03347630637007078, "avg_duration_per_task_ms": 767457.7462082912, "avg_output_tokens_per_task": 38389.49544994944, "last_run_timestamp": "2026-10-24T08:26:00.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8867506138022562, "accuracy_std_dev": 0.020605395430793746, "run_count": 8, "total_questions": 1507, "avg_cost_per_task_usd": 0.1588602813536828, "avg_duration_per_task_ms": 705608.4233576643, "avg_output_tokens_per_task": 51029.565361645655, "last_run_timestamp": "2026-10-24T16:27:11.137Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8867197848605577, "accuracy_std_dev": 0.007952474956944445, "run_count": 13, "total_questions": 2510, "avg_cost_per_task_usd": 0.09700779960159363, "avg_duration_per_task_ms": 461798.00398406375, "avg_output_tokens_per_task": 28989.141832669324, "last_run_timestamp": "2026-10-24T16:06:30.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8866872307692307, "accuracy_std_dev": 0.01902814900716969, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.027383982128982128, "avg_duration_per_task_ms": 210380.49145299144, "avg_output_tokens_per_task": 25687.588966588966, "last_run_timestamp": "2026-10-24T08:19:41.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8865963923784493, "accuracy_std_dev": 0.13919925741876826, "run_count": 21, "total_questions": 3805, "avg_cost_per_task_usd": 0.15561201261498026, "avg_duration_per_task_ms": 231786.43758212877, "avg_output_tokens_per_task": 10794.42838370565, "last_run_timestamp": "2026-10-24T08:53:46.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006986138888888889, "avg_duration_per_task_ms": 141699.11616161617, "avg_output_tokens_per_task": 11730.31313131313, "last_run_timestamp": "2026-10-24T08:29:14.214Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8863540053865651, "accuracy_std_dev": 0.012359654596078922, "run_count": 16, "total_questions": 3156, "avg_cost_per_task_usd": 0.007512178707224334, "avg_duration_per_task_ms": 424002.4524714829, "avg_output_tokens_per_task": 26773.751901140684, "last_run_timestamp": "2026-10-24T08:50:11.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8859760038834952, "accuracy_std_dev": 0.023393307207604786, "run_count": 8, "total_questions": 1545, "avg_cost_per_task_usd": 0.02020688673139159, "avg_duration_per_task_ms": 264412.80194174754, "avg_output_tokens_per_task": 30725.21488673139, "last_run_timestamp": "2026-10-24T09:10:08.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8858024166666666, "accuracy_std_dev": 0.035974938034364815, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1124567276936027, "avg_duration_per_task_ms": 508472.5484006734, "avg_output_tokens_per_task": 36070.853114478115, "last_run_timestamp": "2026-10-24T08:17:53.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.885662, "accuracy_std_dev": 0.014374902133000174, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11714179755892257, "avg_duration_per_task_ms": 572173.5332491583, "avg_output_tokens_per_task": 30582.406986531987, "last_run_timestamp": "2026-10-24T16:30:32.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8856471629778674, "accuracy_std_dev": 0.009013068317726206, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.010023977867203217, "avg_duration_per_task_ms": 270739.96177062375, "avg_output_tokens_per_task": 28220.264922870556, "last_run_timestamp": "2026-10-24T09:01:43.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8855220000000001, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1415003787878788, "avg_duration_per_task_ms": 184472.2676767677, "avg_output_tokens_per_task": 47043.42424242424, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007081343434343435, "avg_duration_per_task_ms": 121564.28282828283, "avg_output_tokens_per_task": 11603.333333333334, "last_run_timestamp": "2026-10-24T08:10:23.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8853190481927712, "accuracy_std_dev": 0.005795847495189989, "run_count": 4, "total_questions": 747, "avg_cost_per_task_usd": 0.0097919718875502, "avg_duration_per_task_ms": 50542.31994645247, "avg_output_tokens_per_task": 7272.5394912985275, "last_run_timestamp": "2026-10-01T19:09:12.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8850689656218401, "accuracy_std_dev": 0.014411932672615443, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.026149528816986855, "avg_duration_per_task_ms": 355797.47320525785, "avg_output_tokens_per_task": 26995.41152679474, "last_run_timestamp": "2026-10-10T08:22:45.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.884890375, "accuracy_std_dev": 0.021787089743340537, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.22976492803030302, "avg_duration_per_task_ms": 949653.6900252525, "avg_output_tokens_per_task": 57582.06123737374, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Together", "accuracy": 0.8846800000000001, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01685389393939394, "avg_duration_per_task_ms": 60063.59848484849, "avg_output_tokens_per_task": 13866.29292929293, "last_run_timestamp": "2026-10-24T08:11:43.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.884566476280835, "accuracy_std_dev": 0.018280900654354407, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.06262646173308033, "avg_duration_per_task_ms": 529742.5496521189, "avg_output_tokens_per_task": 35635.60657811512, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8842948996342949, "accuracy_std_dev": 0.013500101124006438, "run_count": 25, "total_questions": 4922, "avg_cost_per_task_usd": 0.19286014851686306, "avg_duration_per_task_ms": 362955.4989841528, "avg_output_tokens_per_task": 31469.974603819584, "last_run_timestamp": "2026-10-24T16:13:46.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.884269595505618, "accuracy_std_dev": 0.010583520080008851, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.018860816853932583, "avg_duration_per_task_ms": 74171.62584269662, "avg_output_tokens_per_task": 7862.741573033708, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8838383333333335, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.13180945791245793, "avg_duration_per_task_ms": 248970.3754208754, "avg_output_tokens_per_task": 43817.097643097644, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014282979797979798, "avg_duration_per_task_ms": 55075.11111111111, "avg_output_tokens_per_task": 13355.585858585859, "last_run_timestamp": "2026-10-24T08:02:32.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8838218675352877, "accuracy_std_dev": 0.0032754605783004037, "run_count": 5, "total_questions": 921, "avg_cost_per_task_usd": 0.10965064820846905, "avg_duration_per_task_ms": 399019.9750271444, "avg_output_tokens_per_task": 36828.9565689468, "last_run_timestamp": "2026-10-24T16:03:25.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8836519487704917, "accuracy_std_dev": 0.06806651408353726, "run_count": 16, "total_questions": 2928, "avg_cost_per_task_usd": 0.009323294740437157, "avg_duration_per_task_ms": 150365.9463797814, "avg_output_tokens_per_task": 24280.998292349726, "last_run_timestamp": "2026-10-24T08:48:19.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.88341725, "accuracy_std_dev": 0.003834202809907171, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03130394823232323, "avg_duration_per_task_ms": 201934.6616161616, "avg_output_tokens_per_task": 25861.328282828283, "last_run_timestamp": "2026-10-24T08:53:53.931Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8833333472727273, "accuracy_std_dev": 0.012158692807021237, "run_count": 7, "total_questions": 1100, "avg_cost_per_task_usd": 0.007285390000000001, "avg_duration_per_task_ms": 942880.7490909091, "avg_output_tokens_per_task": 28917.175454545453, "last_run_timestamp": "2026-10-17T08:19:36.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.8828761428571429, "accuracy_std_dev": 0.004395219722398795, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.029912125541125538, "avg_duration_per_task_ms": 82923.05122655122, "avg_output_tokens_per_task": 24775.42784992785, "last_run_timestamp": "2026-10-24T08:49:27.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.8828537731629394, "accuracy_std_dev": 0.008622317150279293, "run_count": 5, "total_questions": 939, "avg_cost_per_task_usd": 0.006383692225772098, "avg_duration_per_task_ms": 468696.3525026624, "avg_output_tokens_per_task": 35072.061767838124, "last_run_timestamp": "2026-10-24T08:44:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8821585088105727, "accuracy_std_dev": 0.01851343261661651, "run_count": 14, "total_questions": 2724, "avg_cost_per_task_usd": 0.1276248480176211, "avg_duration_per_task_ms": 62648.52643171806, "avg_output_tokens_per_task": 4878.82268722467, "last_run_timestamp": "2026-10-24T08:27:56.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.8821550000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.129083601010101, "avg_duration_per_task_ms": 388751.6590909091, "avg_output_tokens_per_task": 32112.176767676767, "last_run_timestamp": "2026-10-17T16:07:32.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.88215475, "accuracy_std_dev": 0.018821912135527027, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008138106691919191, "avg_duration_per_task_ms": 380823.72411616164, "avg_output_tokens_per_task": 28681.33712121212, "last_run_timestamp": "2026-10-24T16:29:33.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.882043637444875, "accuracy_std_dev": 0.009663107099304217, "run_count": 39, "total_questions": 7483, "avg_cost_per_task_usd": 0.13849482640652144, "avg_duration_per_task_ms": 62839.13149806228, "avg_output_tokens_per_task": 5015.037551784044, "last_run_timestamp": "2026-10-24T08:22:55.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.882034357142857, "accuracy_std_dev": 0.011301592736090304, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.006003468614718615, "avg_duration_per_task_ms": 401285.20815295813, "avg_output_tokens_per_task": 29705.93326118326, "last_run_timestamp": "2026-10-24T08:46:33.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8818485454545456, "accuracy_std_dev": 0.01327175411002725, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.18638605096418728, "avg_duration_per_task_ms": 321116.0505050505, "avg_output_tokens_per_task": 46783.020661157025, "last_run_timestamp": "2026-10-24T08:02:27.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.8816240000000001, "accuracy_std_dev": 0.004701000000000011, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.020361825641025642, "avg_duration_per_task_ms": 90976.65641025642, "avg_output_tokens_per_task": 40519.597435897434, "last_run_timestamp": "2026-10-24T08:02:32.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.881603, "accuracy_std_dev": null, "run_count": 1, "total_questions": 183, "avg_cost_per_task_usd": 0.10363212021857923, "avg_duration_per_task_ms": 876152.9726775957, "avg_output_tokens_per_task": 41457.83606557377, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8815751316455697, "accuracy_std_dev": 0.010764200383163109, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.007927140084388185, "avg_duration_per_task_ms": 240915.00168776372, "avg_output_tokens_per_task": 27983.259071729957, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.8815374099783081, "accuracy_std_dev": 0.05700823639454502, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.08337883875632683, "avg_duration_per_task_ms": 715683.0332610267, "avg_output_tokens_per_task": 31906.826464208243, "last_run_timestamp": "2026-10-24T16:21:36.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8814552816901408, "accuracy_std_dev": 0.00965776820360461, "run_count": 13, "total_questions": 2556, "avg_cost_per_task_usd": 0.00679829303599374, "avg_duration_per_task_ms": 375447.8978873239, "avg_output_tokens_per_task": 19618.11854460094, "last_run_timestamp": "2026-10-24T08:46:28.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8814329055819478, "accuracy_std_dev": 0.011397141503221075, "run_count": 9, "total_questions": 1684, "avg_cost_per_task_usd": 0.06679124287410926, "avg_duration_per_task_ms": 309462.4631828979, "avg_output_tokens_per_task": 18456.302850356293, "last_run_timestamp": "2026-10-24T16:08:29.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.881313, "accuracy_std_dev": 0.012626000000000026, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.022825318181818183, "avg_duration_per_task_ms": 147723.58080808082, "avg_output_tokens_per_task": 45335.5101010101, "last_run_timestamp": "2026-10-24T08:12:49.570Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8813128749999999, "accuracy_std_dev": 0.02861962867700026, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19908026893939393, "avg_duration_per_task_ms": 958354.9299242424, "avg_output_tokens_per_task": 58904.72348484849, "last_run_timestamp": "2026-10-24T16:17:57.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8809269425684658, "accuracy_std_dev": 0.03229722684698395, "run_count": 19, "total_questions": 3761, "avg_cost_per_task_usd": 0.1444545269875033, "avg_duration_per_task_ms": 371121.18771603296, "avg_output_tokens_per_task": 10937.842329167774, "last_run_timestamp": "2026-10-24T08:22:00.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.8808292124352332, "accuracy_std_dev": 0.0020954999999999724, "run_count": 2, "total_questions": 386, "avg_cost_per_task_usd": 0.03105037564766839, "avg_duration_per_task_ms": 282019.7538860104, "avg_output_tokens_per_task": 61860.47668393782, "last_run_timestamp": "2026-10-24T08:07:42.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8807385965665235, "accuracy_std_dev": 0.04941426514861386, "run_count": 13, "total_questions": 2563, "avg_cost_per_task_usd": 0.0748979442060086, "avg_duration_per_task_ms": 557063.2532188841, "avg_output_tokens_per_task": 39438.98985563793, "last_run_timestamp": "2026-10-24T16:40:43.534Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8800047366136035, "accuracy_std_dev": 0.007207181987717666, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.03244589001447178, "avg_duration_per_task_ms": 148073.17149059335, "avg_output_tokens_per_task": 26851.936324167873, "last_run_timestamp": "2026-10-24T08:26:33.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8794324650455927, "accuracy_std_dev": 0.01587967361881218, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.03493195542046606, "avg_duration_per_task_ms": 723219.4984802431, "avg_output_tokens_per_task": 40176.24822695035, "last_run_timestamp": "2026-10-24T08:41:21.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.879370525058275, "accuracy_std_dev": 0.02189493848211963, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.09564452447552448, "avg_duration_per_task_ms": 551582.8269230769, "avg_output_tokens_per_task": 28335.815268065267, "last_run_timestamp": "2026-10-24T08:13:46.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8792376607706888, "accuracy_std_dev": 0.006337813294770935, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.016976666456096023, "avg_duration_per_task_ms": 306778.0650663297, "avg_output_tokens_per_task": 30476.99052432091, "last_run_timestamp": "2026-10-24T08:43:17.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8790282857142857, "accuracy_std_dev": 0.019161523885971404, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03542920851370852, "avg_duration_per_task_ms": 140156.7987012987, "avg_output_tokens_per_task": 27224.489898989897, "last_run_timestamp": "2026-10-24T08:28:56.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "Deepseek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010643959595959596, "avg_duration_per_task_ms": 127627.76767676767, "avg_output_tokens_per_task": 14277.949494949495, "last_run_timestamp": "2026-10-08T13:29:57.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Mimo V2.6 Pro", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01249009595959596, "avg_duration_per_task_ms": 315799.5808080808, "avg_output_tokens_per_task": 14226.873737373737, "last_run_timestamp": "2026-10-08T01:55:44.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8787876666666666, "accuracy_std_dev": 0.005991714074916775, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007907055555555554, "avg_duration_per_task_ms": 79561.398989899, "avg_output_tokens_per_task": 7834.653198653199, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.8787668916494136, "accuracy_std_dev": 0.03281417517628624, "run_count": 8, "total_questions": 1449, "avg_cost_per_task_usd": 0.2005868881987578, "avg_duration_per_task_ms": 1332066.3747412008, "avg_output_tokens_per_task": 50003.33195307109, "last_run_timestamp": "2026-10-24T16:26:57.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8786913361593894, "accuracy_std_dev": 0.025462712419203403, "run_count": 12, "total_questions": 2359, "avg_cost_per_task_usd": 0.08448664264518864, "avg_duration_per_task_ms": 403631.4383213226, "avg_output_tokens_per_task": 31865.207291225095, "last_run_timestamp": "2026-10-24T16:13:03.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8786261545293074, "accuracy_std_dev": 0.0068412882173039926, "run_count": 6, "total_questions": 1126, "avg_cost_per_task_usd": 0.11316016607460035, "avg_duration_per_task_ms": 476858.84191829484, "avg_output_tokens_per_task": 32318.380994671403, "last_run_timestamp": "2026-10-17T16:07:16.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.87780575, "accuracy_std_dev": 0.016897481104812654, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1286623606902357, "avg_duration_per_task_ms": 62852.628367003366, "avg_output_tokens_per_task": 4923.856481481482, "last_run_timestamp": "2026-10-24T08:15:51.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8776513109987357, "accuracy_std_dev": 0.007333672175410435, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.11767066118836916, "avg_duration_per_task_ms": 752365.9431099873, "avg_output_tokens_per_task": 30843.225874420565, "last_run_timestamp": "2026-10-24T16:07:46.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.8774407999999999, "accuracy_std_dev": 0.00954717737134911, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03921924343434343, "avg_duration_per_task_ms": 242470.04242424242, "avg_output_tokens_per_task": 37354.009090909094, "last_run_timestamp": "2026-10-21T11:23:15.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8773447142857143, "accuracy_std_dev": 0.012707857195050472, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03401366883116883, "avg_duration_per_task_ms": 154330.79004329004, "avg_output_tokens_per_task": 28188.37950937951, "last_run_timestamp": "2026-10-24T09:09:44.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8770437482014388, "accuracy_std_dev": 0.016843957465790968, "run_count": 8, "total_questions": 1529, "avg_cost_per_task_usd": 0.1855886553302812, "avg_duration_per_task_ms": 777107.6200130804, "avg_output_tokens_per_task": 53473.10595160235, "last_run_timestamp": "2026-10-24T16:28:49.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.876999, "accuracy_std_dev": 0.06406231714705611, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1307801066919192, "avg_duration_per_task_ms": 1601351.4103535353, "avg_output_tokens_per_task": 55537.77272727273, "last_run_timestamp": "2026-10-24T16:02:47.376Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8769172222222222, "accuracy_std_dev": 0.01657946057812884, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02028404994388328, "avg_duration_per_task_ms": 725618.4191919192, "avg_output_tokens_per_task": 35595.71436588103, "last_run_timestamp": "2026-10-24T16:27:17.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.8767675999999999, "accuracy_std_dev": 0.014101374310328764, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.042260786868686874, "avg_duration_per_task_ms": 210147.5181818182, "avg_output_tokens_per_task": 35072.08888888889, "last_run_timestamp": "2026-10-24T08:50:45.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8765007667238421, "accuracy_std_dev": 0.009087011145341243, "run_count": 3, "total_questions": 583, "avg_cost_per_task_usd": 0.09344886277873071, "avg_duration_per_task_ms": 568169.3550600343, "avg_output_tokens_per_task": 27396.61578044597, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.8763533692307693, "accuracy_std_dev": 0.006821595577445394, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.027450420512820508, "avg_duration_per_task_ms": 188376.252991453, "avg_output_tokens_per_task": 30391.492307692308, "last_run_timestamp": "2026-10-23T06:17:23.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8763108617977529, "accuracy_std_dev": 0.006823199452268141, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.12380380168539326, "avg_duration_per_task_ms": 917825.4601123596, "avg_output_tokens_per_task": 43434.7488764045, "last_run_timestamp": "2026-10-24T16:19:29.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8760746752655538, "accuracy_std_dev": 0.011470076619186123, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.009704542235710673, "avg_duration_per_task_ms": 429066.8811330298, "avg_output_tokens_per_task": 22619.601416287303, "last_run_timestamp": "2026-10-24T08:41:28.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8760220000000001, "accuracy_std_dev": 0.014151103773204405, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.11097613275613274, "avg_duration_per_task_ms": 457617.5707070707, "avg_output_tokens_per_task": 31565.57683982684, "last_run_timestamp": "2026-10-24T16:02:47.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04156198232323233, "avg_duration_per_task_ms": 309570.8888888889, "avg_output_tokens_per_task": 39159.9494949495, "last_run_timestamp": "2026-10-17T08:26:18.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01960839393939394, "avg_duration_per_task_ms": 110709.8308080808, "avg_output_tokens_per_task": 43264.59848484849, "last_run_timestamp": "2026-10-24T08:12:37.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8752913076923076, "accuracy_std_dev": 0.04445532892411749, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.09291413209013209, "avg_duration_per_task_ms": 618152.5388500389, "avg_output_tokens_per_task": 41235.80536130536, "last_run_timestamp": "2026-10-24T16:04:29.298Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.8749675384615386, "accuracy_std_dev": 0.013017813792677657, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.006246133255633255, "avg_duration_per_task_ms": 265528.69425019424, "avg_output_tokens_per_task": 18684.931235431235, "last_run_timestamp": "2026-10-24T08:20:25.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8749014196217493, "accuracy_std_dev": 0.02259480542425625, "run_count": 5, "total_questions": 846, "avg_cost_per_task_usd": 0.12702199408983453, "avg_duration_per_task_ms": 548158.9716312056, "avg_output_tokens_per_task": 48997.07092198582, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8747894999999999, "accuracy_std_dev": 0.02495521020748971, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1433024273989899, "avg_duration_per_task_ms": 926726.0309343435, "avg_output_tokens_per_task": 61552.77146464647, "last_run_timestamp": "2026-10-24T16:07:10.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8747014285714287, "accuracy_std_dev": 0.09649629293840344, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.02719648798988622, "avg_duration_per_task_ms": 204176.03792667508, "avg_output_tokens_per_task": 23353.09228824273, "last_run_timestamp": "2026-10-24T16:28:20.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.8746087667870037, "accuracy_std_dev": 0.007592482877484693, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.2179415407942238, "avg_duration_per_task_ms": 1239989.4288808664, "avg_output_tokens_per_task": 48485.83321299639, "last_run_timestamp": "2026-10-24T16:18:35.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.8745790000000001, "accuracy_std_dev": 0.04079651270022963, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.010708270202020202, "avg_duration_per_task_ms": 125365.15404040404, "avg_output_tokens_per_task": 15791.75, "last_run_timestamp": "2026-10-24T08:20:32.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.874579, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018065838383838382, "avg_duration_per_task_ms": 168823.5202020202, "avg_output_tokens_per_task": 29810.62373737374, "last_run_timestamp": "2026-10-24T08:41:24.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8744854444444443, "accuracy_std_dev": 0.005088649135545852, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.08018900897867563, "avg_duration_per_task_ms": 386083.189674523, "avg_output_tokens_per_task": 22800.944444444445, "last_run_timestamp": "2026-10-24T16:39:19.451Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8743608677867055, "accuracy_std_dev": 0.030009909369776877, "run_count": 15, "total_questions": 2738, "avg_cost_per_task_usd": 0.011979280861943022, "avg_duration_per_task_ms": 340619.6665449233, "avg_output_tokens_per_task": 21211.479547114683, "last_run_timestamp": "2026-10-24T08:45:03.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.8742983333333334, "accuracy_std_dev": 0.009152249171044747, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0078061279461279455, "avg_duration_per_task_ms": 622665.6515151515, "avg_output_tokens_per_task": 44426.93771043771, "last_run_timestamp": "2026-10-24T16:20:24.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.8742830219449225, "accuracy_std_dev": 0.06426649201755963, "run_count": 12, "total_questions": 2324, "avg_cost_per_task_usd": 0.01125052065404475, "avg_duration_per_task_ms": 484759.71815834765, "avg_output_tokens_per_task": 23203.48364888124, "last_run_timestamp": "2026-10-24T08:11:11.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8740843414084506, "accuracy_std_dev": 0.004005737471597214, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.044288470985915485, "avg_duration_per_task_ms": 203522.5785915493, "avg_output_tokens_per_task": 38212.04394366197, "last_run_timestamp": "2026-10-24T08:44:12.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8740044607508533, "accuracy_std_dev": 0.007605951280996289, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.10401818771331058, "avg_duration_per_task_ms": 459122.319112628, "avg_output_tokens_per_task": 30022.856655290103, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8739276971507354, "accuracy_std_dev": 0.014730569376258348, "run_count": 11, "total_questions": 2176, "avg_cost_per_task_usd": 0.09131274172794118, "avg_duration_per_task_ms": 205017.0390625, "avg_output_tokens_per_task": 23097.8125, "last_run_timestamp": "2026-10-24T08:48:36.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8737371428571428, "accuracy_std_dev": 0.01647031431245381, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.008272517676767677, "avg_duration_per_task_ms": 219776.6417748918, "avg_output_tokens_per_task": 29274.76370851371, "last_run_timestamp": "2026-10-24T09:11:12.594Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.8737370000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03531422474747475, "avg_duration_per_task_ms": 145858.30808080808, "avg_output_tokens_per_task": 29116.540404040403, "last_run_timestamp": "2026-10-24T08:59:28.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.8737370000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023037462121212122, "avg_duration_per_task_ms": 506854.46717171714, "avg_output_tokens_per_task": 48027.71464646464, "last_run_timestamp": "2026-10-24T08:01:41.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.873716393364929, "accuracy_std_dev": 0.03410588644533702, "run_count": 9, "total_questions": 1688, "avg_cost_per_task_usd": 0.012903949052132702, "avg_duration_per_task_ms": 360645.17417061614, "avg_output_tokens_per_task": 25970.889218009477, "last_run_timestamp": "2026-10-24T08:43:07.915Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8736468057110863, "accuracy_std_dev": 0.03036521292219109, "run_count": 10, "total_questions": 1786, "avg_cost_per_task_usd": 0.025540696528555434, "avg_duration_per_task_ms": 360273.59238521836, "avg_output_tokens_per_task": 24510.389137737962, "last_run_timestamp": "2026-10-24T16:24:04.274Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8733163333333334, "accuracy_std_dev": 0.030039827993663497, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.17326605218855218, "avg_duration_per_task_ms": 357944.43055555556, "avg_output_tokens_per_task": 39522.67929292929, "last_run_timestamp": "2026-10-24T16:27:20.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8732174942820837, "accuracy_std_dev": 0.03602354128311638, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.10224943286742906, "avg_duration_per_task_ms": 640310.1185938162, "avg_output_tokens_per_task": 25410.69970351546, "last_run_timestamp": "2026-10-24T16:26:55.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8727272475524476, "accuracy_std_dev": 0.029233686851601524, "run_count": 8, "total_questions": 1430, "avg_cost_per_task_usd": 0.09700304615384615, "avg_duration_per_task_ms": 194874.37272727274, "avg_output_tokens_per_task": 26397.99300699301, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8724745, "accuracy_std_dev": 0.008858340547190563, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008325782828282828, "avg_duration_per_task_ms": 77307.88005050505, "avg_output_tokens_per_task": 7842.704545454545, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8723343333333334, "accuracy_std_dev": 0.005338803632109208, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06334109932659931, "avg_duration_per_task_ms": 262990.18855218857, "avg_output_tokens_per_task": 21026.934343434343, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8721539397539244, "accuracy_std_dev": 0.008970575607073516, "run_count": 12, "total_questions": 2357, "avg_cost_per_task_usd": 0.008344311412812899, "avg_duration_per_task_ms": 339433.18073822657, "avg_output_tokens_per_task": 29494.21552821383, "last_run_timestamp": "2026-10-17T08:23:51.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.872053625, "accuracy_std_dev": 0.02370397511989021, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1190935284090909, "avg_duration_per_task_ms": 606301.1369949495, "avg_output_tokens_per_task": 35231.555555555555, "last_run_timestamp": "2026-10-24T16:26:49.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8720033968503939, "accuracy_std_dev": 0.01337304654930955, "run_count": 10, "total_questions": 1905, "avg_cost_per_task_usd": 0.008342944356955382, "avg_duration_per_task_ms": 238567.13018372704, "avg_output_tokens_per_task": 29657.869816272967, "last_run_timestamp": "2026-10-24T16:02:06.915Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8717731666666667, "accuracy_std_dev": 0.004601771648567064, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.00424808627946128, "avg_duration_per_task_ms": 39417.8244949495, "avg_output_tokens_per_task": 7731.690235690236, "last_run_timestamp": "2026-10-24T08:30:49.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8714859264426126, "accuracy_std_dev": 0.00752103815972237, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.06945589727330374, "avg_duration_per_task_ms": 276858.71528218134, "avg_output_tokens_per_task": 11308.629676601142, "last_run_timestamp": "2026-10-10T08:22:14.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8713054444444444, "accuracy_std_dev": 0.011226478265552096, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.08585465039281706, "avg_duration_per_task_ms": 191685.55331088664, "avg_output_tokens_per_task": 13992.872615039281, "last_run_timestamp": "2026-10-24T08:29:16.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8712332981471084, "accuracy_std_dev": 0.008224257022992851, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.049519284671532854, "avg_duration_per_task_ms": 283250.0583941606, "avg_output_tokens_per_task": 21034.03425042111, "last_run_timestamp": "2026-10-24T16:29:09.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.871212, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017874631313131314, "avg_duration_per_task_ms": 86992.41414141415, "avg_output_tokens_per_task": 29675.651515151516, "last_run_timestamp": "2026-10-24T08:40:56.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8711184444444445, "accuracy_std_dev": 0.018965626509914724, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.19942039898989897, "avg_duration_per_task_ms": 553312.7272727273, "avg_output_tokens_per_task": 58346.15095398429, "last_run_timestamp": "2026-10-24T16:09:40.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.8710915714285715, "accuracy_std_dev": 0.014505946572326232, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.021487838383838383, "avg_duration_per_task_ms": 272335.1349206349, "avg_output_tokens_per_task": 35144.1277056277, "last_run_timestamp": "2026-10-24T08:13:46.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8706863196512894, "accuracy_std_dev": 0.010805462536728977, "run_count": 14, "total_questions": 2753, "avg_cost_per_task_usd": 0.047070880130766435, "avg_duration_per_task_ms": 208861.94733018524, "avg_output_tokens_per_task": 36299.43697784236, "last_run_timestamp": "2026-10-24T09:06:03.113Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8703702499999999, "accuracy_std_dev": 0.005050541672682263, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00817759595959596, "avg_duration_per_task_ms": 373579.7626262626, "avg_output_tokens_per_task": 28859.406565656565, "last_run_timestamp": "2026-10-24T16:12:07.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.86994925, "accuracy_std_dev": 0.017018176890827654, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03481110921717172, "avg_duration_per_task_ms": 505625.7146464646, "avg_output_tokens_per_task": 35174.42866161616, "last_run_timestamp": "2026-10-24T16:26:47.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8698803797766019, "accuracy_std_dev": 0.006846731845194459, "run_count": 10, "total_questions": 1701, "avg_cost_per_task_usd": 0.05467326690182246, "avg_duration_per_task_ms": 430735.48677248677, "avg_output_tokens_per_task": 18124.479129923573, "last_run_timestamp": "2026-10-24T16:13:46.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.869809111111111, "accuracy_std_dev": 0.008765689330115383, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07596271436588103, "avg_duration_per_task_ms": 102049.09147025814, "avg_output_tokens_per_task": 12345.802469135802, "last_run_timestamp": "2026-10-24T08:49:10.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8697483287395923, "accuracy_std_dev": 0.025647694972356895, "run_count": 18, "total_questions": 3483, "avg_cost_per_task_usd": 0.03920849038185472, "avg_duration_per_task_ms": 78569.80017226528, "avg_output_tokens_per_task": 25460.256675279932, "last_run_timestamp": "2026-10-24T08:27:04.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8694541060070672, "accuracy_std_dev": 0.017661177225418892, "run_count": 9, "total_questions": 1698, "avg_cost_per_task_usd": 0.07418639340400471, "avg_duration_per_task_ms": 396957.8833922261, "avg_output_tokens_per_task": 24048.35217903416, "last_run_timestamp": "2026-10-24T08:28:10.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8692479999999999, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22171807575757574, "avg_duration_per_task_ms": 260359.50673400675, "avg_output_tokens_per_task": 22076.11111111111, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.8691077499999998, "accuracy_std_dev": 0.005877315899924065, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.003982401515151515, "avg_duration_per_task_ms": 37577.292929292926, "avg_output_tokens_per_task": 7724.75, "last_run_timestamp": "2026-10-24T08:16:38.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8689927272727271, "accuracy_std_dev": 0.03917244891523285, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.17977295500459137, "avg_duration_per_task_ms": 45320.05463728191, "avg_output_tokens_per_task": 3392.5229568411387, "last_run_timestamp": "2026-10-24T08:48:02.790Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.868687, "accuracy_std_dev": 0.006299080210633947, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004035915404040404, "avg_duration_per_task_ms": 38787.454545454544, "avg_output_tokens_per_task": 7838.214646464647, "last_run_timestamp": "2026-10-24T08:07:26.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024268303030303032, "avg_duration_per_task_ms": 252794.202020202, "avg_output_tokens_per_task": 20048.777777777777, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.005583575528950976, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004182061868686869, "avg_duration_per_task_ms": 36016.292929292926, "avg_output_tokens_per_task": 7729.580176767677, "last_run_timestamp": "2026-10-24T08:19:41.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.016015464836760137, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.038159512626262625, "avg_duration_per_task_ms": 691231.6849747475, "avg_output_tokens_per_task": 34768.117424242424, "last_run_timestamp": "2026-10-24T16:16:05.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.8686866666666666, "accuracy_std_dev": 0.004955863351178644, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004029786195286195, "avg_duration_per_task_ms": 38652.87037037037, "avg_output_tokens_per_task": 7824.787878787879, "last_run_timestamp": "2026-10-24T08:01:41.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.8686865, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01774992424242424, "avg_duration_per_task_ms": 402650.36616161617, "avg_output_tokens_per_task": 29383.83585858586, "last_run_timestamp": "2026-10-24T08:28:10.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8686612136752137, "accuracy_std_dev": 0.02320764839922872, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.025878321367521367, "avg_duration_per_task_ms": 786906.9829059829, "avg_output_tokens_per_task": 23783.8, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.868376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.027671912820512817, "avg_duration_per_task_ms": 285303.4820512821, "avg_output_tokens_per_task": 21385.076923076922, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8682876231003039, "accuracy_std_dev": 0.028807543557374, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.10277313323201621, "avg_duration_per_task_ms": 355978.4974670719, "avg_output_tokens_per_task": 23116.26595744681, "last_run_timestamp": "2026-10-24T08:57:08.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8682058571428571, "accuracy_std_dev": 0.007673379706651383, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07688844300144301, "avg_duration_per_task_ms": 83582.13492063493, "avg_output_tokens_per_task": 12574.14357864358, "last_run_timestamp": "2026-10-10T08:21:46.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8676766000000001, "accuracy_std_dev": 0.004205309600968765, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07033428585858588, "avg_duration_per_task_ms": 232782.28383838382, "avg_output_tokens_per_task": 11454.874242424243, "last_run_timestamp": "2026-10-24T08:34:00.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.867634625, "accuracy_std_dev": 0.007431114989311851, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025694838383838382, "avg_duration_per_task_ms": 248942.36553030304, "avg_output_tokens_per_task": 21167.008207070707, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8675645, "accuracy_std_dev": 0.008228295808772726, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.004289666245791245, "avg_duration_per_task_ms": 35945.74074074074, "avg_output_tokens_per_task": 7815.2925084175085, "last_run_timestamp": "2026-10-24T08:43:41.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.867003, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013525050505050505, "avg_duration_per_task_ms": 181228.28787878787, "avg_output_tokens_per_task": 22673.38888888889, "last_run_timestamp": "2026-10-24T08:02:32.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8666971818181819, "accuracy_std_dev": 0.02628554379335525, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.007686401744719926, "avg_duration_per_task_ms": 446263.08815426996, "avg_output_tokens_per_task": 25752.568411386594, "last_run_timestamp": "2026-10-24T08:15:15.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8664308674130818, "accuracy_std_dev": 0.027802409922633914, "run_count": 9, "total_questions": 1697, "avg_cost_per_task_usd": 0.011504347672362995, "avg_duration_per_task_ms": 435596.48084855627, "avg_output_tokens_per_task": 25872.670005892753, "last_run_timestamp": "2026-10-24T08:11:43.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.8660681111111111, "accuracy_std_dev": 0.016724420165630353, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07373856228956228, "avg_duration_per_task_ms": 287527.3310886644, "avg_output_tokens_per_task": 20979.62962962963, "last_run_timestamp": "2026-10-24T16:08:41.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8653352747021724, "accuracy_std_dev": 0.017080671231962035, "run_count": 8, "total_questions": 1427, "avg_cost_per_task_usd": 0.005114180098107919, "avg_duration_per_task_ms": 702127.0336370007, "avg_output_tokens_per_task": 28117.81079187106, "last_run_timestamp": "2026-10-24T16:15:47.409Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.86532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01495640909090909, "avg_duration_per_task_ms": 120441.0101010101, "avg_output_tokens_per_task": 29795.70707070707, "last_run_timestamp": "2026-10-24T15:00:02.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.865101093922652, "accuracy_std_dev": 0.005917000000000006, "run_count": 2, "total_questions": 362, "avg_cost_per_task_usd": 0.005541756906077348, "avg_duration_per_task_ms": 151652.73480662983, "avg_output_tokens_per_task": 19668.124309392264, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8645117600000001, "accuracy_std_dev": 0.02767954650102491, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.15895250323232327, "avg_duration_per_task_ms": 153406.69333333333, "avg_output_tokens_per_task": 15378.712121212122, "last_run_timestamp": "2026-10-24T08:20:16.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.8644780000000001, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05228046464646464, "avg_duration_per_task_ms": 195135.57323232322, "avg_output_tokens_per_task": 43383.04797979798, "last_run_timestamp": "2026-10-24T08:15:15.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.86433775, "accuracy_std_dev": 0.014059447115285145, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7497476914983165, "avg_duration_per_task_ms": 499048.3042929293, "avg_output_tokens_per_task": 29826.423821548822, "last_run_timestamp": "2026-10-24T08:11:56.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8642836153846152, "accuracy_std_dev": 0.035867989623357806, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.18533466744366744, "avg_duration_per_task_ms": 45806.41025641026, "avg_output_tokens_per_task": 3500.1981351981353, "last_run_timestamp": "2026-10-24T08:45:03.898Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8638953076923075, "accuracy_std_dev": 0.040435466544539306, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.1779076884226884, "avg_duration_per_task_ms": 43919.818181818184, "avg_output_tokens_per_task": 3351.4957264957266, "last_run_timestamp": "2026-10-24T08:27:26.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.863636, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.22125699158249157, "avg_duration_per_task_ms": 259075.24915824915, "avg_output_tokens_per_task": 22030.511784511786, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8635951463414634, "accuracy_std_dev": 0.01668149999999996, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.047886756097560976, "avg_duration_per_task_ms": 401842.43631436315, "avg_output_tokens_per_task": 25387.241192411922, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8635504804532578, "accuracy_std_dev": 0.013176767872113233, "run_count": 9, "total_questions": 1765, "avg_cost_per_task_usd": 0.07539817337110481, "avg_duration_per_task_ms": 281436.6181303116, "avg_output_tokens_per_task": 20817.862322946177, "last_run_timestamp": "2026-10-24T16:12:49.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8634960000000002, "accuracy_std_dev": 0.01008821268444846, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8421511931818182, "avg_duration_per_task_ms": 412180.98316498316, "avg_output_tokens_per_task": 33494.958333333336, "last_run_timestamp": "2026-10-24T08:26:00.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8634435114503817, "accuracy_std_dev": 0.01366835982067709, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.13531842366412217, "avg_duration_per_task_ms": 289599.17048346053, "avg_output_tokens_per_task": 28034.88358778626, "last_run_timestamp": "2026-10-17T08:23:25.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.862963, "accuracy_std_dev": 0.009134508372101892, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06279948181818182, "avg_duration_per_task_ms": 441139.7515151515, "avg_output_tokens_per_task": 56049.472727272725, "last_run_timestamp": "2026-10-24T16:14:38.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8629629000000001, "accuracy_std_dev": 0.053333789303686284, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10982828333333332, "avg_duration_per_task_ms": 474004.9994949495, "avg_output_tokens_per_task": 27746.173737373738, "last_run_timestamp": "2026-10-24T16:35:28.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8629348333333335, "accuracy_std_dev": 0.008541693195080755, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8005278598484848, "avg_duration_per_task_ms": 459239.8367003367, "avg_output_tokens_per_task": 31848.97138047138, "last_run_timestamp": "2026-10-24T08:27:05.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8626261, "accuracy_std_dev": 0.011472416183611877, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07024428484848486, "avg_duration_per_task_ms": 231811.16010101011, "avg_output_tokens_per_task": 11437.808585858586, "last_run_timestamp": "2026-10-24T08:31:24.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8625841249999999, "accuracy_std_dev": 0.006783236993454885, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004199931818181818, "avg_duration_per_task_ms": 250234.78724747474, "avg_output_tokens_per_task": 29101.075757575756, "last_run_timestamp": "2026-10-24T16:02:06.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8625343605313093, "accuracy_std_dev": 0.061262981528279976, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.2509589614168248, "avg_duration_per_task_ms": 1179158.6533839342, "avg_output_tokens_per_task": 55775.307400379505, "last_run_timestamp": "2026-10-24T16:03:06.207Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.861993011958763, "accuracy_std_dev": 0.030079142720378513, "run_count": 13, "total_questions": 2425, "avg_cost_per_task_usd": 0.02418231587628866, "avg_duration_per_task_ms": 455044.95463917527, "avg_output_tokens_per_task": 23028.103917525772, "last_run_timestamp": "2026-10-17T08:18:04.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8619528181818182, "accuracy_std_dev": 0.017198547374273336, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09933105968778695, "avg_duration_per_task_ms": 470423.74885215797, "avg_output_tokens_per_task": 26261.255280073463, "last_run_timestamp": "2026-10-24T16:12:09.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8619527499999999, "accuracy_std_dev": 0.02717181928832702, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13016119191919193, "avg_duration_per_task_ms": 482692.7720959596, "avg_output_tokens_per_task": 34118.55934343435, "last_run_timestamp": "2026-10-24T16:05:51.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8617843, "accuracy_std_dev": 0.016622379589276627, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09082832878787879, "avg_duration_per_task_ms": 184194.31868686868, "avg_output_tokens_per_task": 25099.163131313133, "last_run_timestamp": "2026-10-24T16:20:59.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.861708258064516, "accuracy_std_dev": 0.012137624210372552, "run_count": 6, "total_questions": 1116, "avg_cost_per_task_usd": 0.02452295071684588, "avg_duration_per_task_ms": 145394.43727598566, "avg_output_tokens_per_task": 37540.51523297491, "last_run_timestamp": "2026-10-24T08:12:02.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8616797297297295, "accuracy_std_dev": 0.027641986931017132, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.1728344180999181, "avg_duration_per_task_ms": 158861.68086268086, "avg_output_tokens_per_task": 16129.410865410866, "last_run_timestamp": "2026-10-24T08:48:23.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8614478, "accuracy_std_dev": 0.008075538394435377, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.06555243737373737, "avg_duration_per_task_ms": 217588.9297979798, "avg_output_tokens_per_task": 10657.188383838384, "last_run_timestamp": "2026-10-24T08:11:43.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8613514285714288, "accuracy_std_dev": 0.011656715536635543, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.842712674963925, "avg_duration_per_task_ms": 456401.95995670994, "avg_output_tokens_per_task": 33529.13383838384, "last_run_timestamp": "2026-10-24T08:15:51.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.861111, "accuracy_std_dev": 0.04797999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021780800505050504, "avg_duration_per_task_ms": 138470.81818181818, "avg_output_tokens_per_task": 20189.02525252525, "last_run_timestamp": "2026-10-24T08:12:37.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.861111, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01767211111111111, "avg_duration_per_task_ms": 125210.21717171717, "avg_output_tokens_per_task": 29259.424242424244, "last_run_timestamp": "2026-10-24T08:13:46.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8609707916666667, "accuracy_std_dev": 0.007924941871391254, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.8830173162878786, "avg_duration_per_task_ms": 414862.5126262626, "avg_output_tokens_per_task": 33448.23737373737, "last_run_timestamp": "2026-10-24T08:44:41.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8608923543307087, "accuracy_std_dev": 0.017439499999999997, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.003837814960629921, "avg_duration_per_task_ms": 298144.1771653543, "avg_output_tokens_per_task": 11523.807086614173, "last_run_timestamp": "2026-10-03T17:04:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8605965053212455, "accuracy_std_dev": 0.03960090537740342, "run_count": 13, "total_questions": 2537, "avg_cost_per_task_usd": 0.01657229877808435, "avg_duration_per_task_ms": 463843.5068979109, "avg_output_tokens_per_task": 41792.180134016555, "last_run_timestamp": "2026-10-24T08:58:59.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8602692499999999, "accuracy_std_dev": 0.07328871721500406, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0069893973063973065, "avg_duration_per_task_ms": 603844.6805555555, "avg_output_tokens_per_task": 28814.18181818182, "last_run_timestamp": "2026-10-24T08:23:31.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.860269, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00987439393939394, "avg_duration_per_task_ms": 74831.28787878787, "avg_output_tokens_per_task": 9997.058080808081, "last_run_timestamp": "2026-10-24T08:14:36.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8601809488313328, "accuracy_std_dev": 0.03964118133468496, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.16196978711307647, "avg_duration_per_task_ms": 990576.0770688566, "avg_output_tokens_per_task": 55234.00947567909, "last_run_timestamp": "2026-10-24T16:06:09.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8600102307692308, "accuracy_std_dev": 0.03934806390536378, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.18072842657342658, "avg_duration_per_task_ms": 45766.77855477855, "avg_output_tokens_per_task": 3406.542735042735, "last_run_timestamp": "2026-10-24T08:08:40.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8599885833333334, "accuracy_std_dev": 0.009617771098842076, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8533179797979797, "avg_duration_per_task_ms": 482512.8122895623, "avg_output_tokens_per_task": 33941.89983164983, "last_run_timestamp": "2026-10-24T08:14:51.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8598806153846154, "accuracy_std_dev": 0.030036815288878832, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.15524810489510485, "avg_duration_per_task_ms": 152128.8811188811, "avg_output_tokens_per_task": 15307.565656565657, "last_run_timestamp": "2026-10-24T08:12:37.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8597419723629663, "accuracy_std_dev": 0.016505636989518208, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.08549684339014277, "avg_duration_per_task_ms": 179079.73422385997, "avg_output_tokens_per_task": 24385.910640257945, "last_run_timestamp": "2026-10-24T08:41:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8596613057736721, "accuracy_std_dev": 0.0311431999096363, "run_count": 11, "total_questions": 2165, "avg_cost_per_task_usd": 0.1269020115473441, "avg_duration_per_task_ms": 528998.0508083141, "avg_output_tokens_per_task": 43596.739953810626, "last_run_timestamp": "2026-10-24T16:12:49.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8594274166666667, "accuracy_std_dev": 0.018601260147538098, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7585617697811449, "avg_duration_per_task_ms": 508230.9494949495, "avg_output_tokens_per_task": 30213.497474747473, "last_run_timestamp": "2026-10-24T08:10:23.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8592592, "accuracy_std_dev": 0.0373207358952098, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.12732033636363638, "avg_duration_per_task_ms": 630447.8414141415, "avg_output_tokens_per_task": 42741.22323232323, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8586479490968801, "accuracy_std_dev": 0.00913590390487393, "run_count": 22, "total_questions": 3654, "avg_cost_per_task_usd": 0.17185495894909686, "avg_duration_per_task_ms": 250875.80268199233, "avg_output_tokens_per_task": 31401.503831417624, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8586340031847132, "accuracy_std_dev": 0.05344060455646062, "run_count": 10, "total_questions": 1884, "avg_cost_per_task_usd": 0.0115320483014862, "avg_duration_per_task_ms": 468303.98089171975, "avg_output_tokens_per_task": 23774.996815286624, "last_run_timestamp": "2026-10-24T08:32:58.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.85858575, "accuracy_std_dev": 0.021526546714173627, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1476876622474748, "avg_duration_per_task_ms": 1253876.004419192, "avg_output_tokens_per_task": 52218.26641414141, "last_run_timestamp": "2026-10-24T16:26:15.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8585855, "accuracy_std_dev": 0.012710146271778291, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014605150252525251, "avg_duration_per_task_ms": 477497.9494949495, "avg_output_tokens_per_task": 26944.89898989899, "last_run_timestamp": "2026-10-24T08:15:32.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.858164875, "accuracy_std_dev": 0.016185868044667075, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1437290511363636, "avg_duration_per_task_ms": 908116.1098484849, "avg_output_tokens_per_task": 45706.13320707071, "last_run_timestamp": "2026-10-24T16:14:06.777Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8577439285714286, "accuracy_std_dev": 0.016868073763619186, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.7558037463924961, "avg_duration_per_task_ms": 505406.40187590185, "avg_output_tokens_per_task": 30123.084054834057, "last_run_timestamp": "2026-10-24T08:30:49.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8576738333333334, "accuracy_std_dev": 0.01763649745004817, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.8048634583333334, "avg_duration_per_task_ms": 484140.92781986535, "avg_output_tokens_per_task": 30507.85164141414, "last_run_timestamp": "2026-10-24T08:15:15.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.857663467955239, "accuracy_std_dev": 0.05434631352575813, "run_count": 10, "total_questions": 1966, "avg_cost_per_task_usd": 0.0781413972533062, "avg_duration_per_task_ms": 509131.62817904376, "avg_output_tokens_per_task": 21110.80518819939, "last_run_timestamp": "2026-10-24T08:57:31.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8576418156329653, "accuracy_std_dev": 0.007837726493074803, "run_count": 18, "total_questions": 3531, "avg_cost_per_task_usd": 0.06304419881053526, "avg_duration_per_task_ms": 588364.0832625319, "avg_output_tokens_per_task": 34740.37921268762, "last_run_timestamp": "2026-10-24T16:03:59.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8573231666666665, "accuracy_std_dev": 0.0170736054571246, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8399580702861953, "avg_duration_per_task_ms": 458034.78324915824, "avg_output_tokens_per_task": 33422.68813131313, "last_run_timestamp": "2026-10-24T08:13:46.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8571887570077085, "accuracy_std_dev": 0.03653443070150671, "run_count": 33, "total_questions": 5708, "avg_cost_per_task_usd": 0.015801940259285212, "avg_duration_per_task_ms": 401037.0720042046, "avg_output_tokens_per_task": 17993.8771899089, "last_run_timestamp": "2026-10-24T08:49:56.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8571427108013937, "accuracy_std_dev": 0.01011902539224449, "run_count": 3, "total_questions": 574, "avg_cost_per_task_usd": 0.11599589721254357, "avg_duration_per_task_ms": 363022.99128919863, "avg_output_tokens_per_task": 29391.742160278747, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8570552727272727, "accuracy_std_dev": 0.02678680556845074, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8339512213039484, "avg_duration_per_task_ms": 436211.7764003673, "avg_output_tokens_per_task": 33251.8347107438, "last_run_timestamp": "2026-10-24T08:08:52.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8569032243034418, "accuracy_std_dev": 0.036552330331727854, "run_count": 22, "total_questions": 4271, "avg_cost_per_task_usd": 0.0041591236244439235, "avg_duration_per_task_ms": 82317.1749004917, "avg_output_tokens_per_task": 15615.828377429174, "last_run_timestamp": "2026-10-24T08:41:46.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.8569025, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018898722222222223, "avg_duration_per_task_ms": 164757.63383838383, "avg_output_tokens_per_task": 31337.035353535353, "last_run_timestamp": "2026-10-24T08:17:56.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8568232764406386, "accuracy_std_dev": 0.020345851767133397, "run_count": 44, "total_questions": 8642, "avg_cost_per_task_usd": 0.2801212903263133, "avg_duration_per_task_ms": 334492.7875491784, "avg_output_tokens_per_task": 50394.622193936586, "last_run_timestamp": "2026-10-24T16:37:01.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8568129407236336, "accuracy_std_dev": 0.01596270457778066, "run_count": 8, "total_questions": 1299, "avg_cost_per_task_usd": 0.004350760585065434, "avg_duration_per_task_ms": 410967.9938414165, "avg_output_tokens_per_task": 9402.565819861431, "last_run_timestamp": "2026-10-24T08:24:10.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.856442089983022, "accuracy_std_dev": 0.021498967023132247, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.04329065647990945, "avg_duration_per_task_ms": 650523.3395585739, "avg_output_tokens_per_task": 36107.27447651386, "last_run_timestamp": "2026-10-24T16:04:33.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8563635600000002, "accuracy_std_dev": 0.050198181182254005, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.18820920787878787, "avg_duration_per_task_ms": 41314.649292929294, "avg_output_tokens_per_task": 3389.456767676768, "last_run_timestamp": "2026-10-24T08:44:43.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.856164284931507, "accuracy_std_dev": 0.05295168533540149, "run_count": 8, "total_questions": 1460, "avg_cost_per_task_usd": 0.18168324041095887, "avg_duration_per_task_ms": 1626314.2294520547, "avg_output_tokens_per_task": 52899.55684931507, "last_run_timestamp": "2026-10-24T16:16:05.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8558499999999999, "accuracy_std_dev": 0.03281670770735542, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.015880464015151512, "avg_duration_per_task_ms": 344363.2039141414, "avg_output_tokens_per_task": 28437.20138888889, "last_run_timestamp": "2026-10-24T16:21:33.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8558169770378875, "accuracy_std_dev": 0.018638102580532955, "run_count": 9, "total_questions": 1742, "avg_cost_per_task_usd": 0.08209278185993112, "avg_duration_per_task_ms": 337904.9672789897, "avg_output_tokens_per_task": 28132.247990815154, "last_run_timestamp": "2026-10-24T08:19:59.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8554993333333334, "accuracy_std_dev": 0.009394238245979377, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005048751683501684, "avg_duration_per_task_ms": 532489.6868686868, "avg_output_tokens_per_task": 30189.478956228955, "last_run_timestamp": "2026-10-24T16:02:08.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.85530295, "accuracy_std_dev": 0.01065364351513132, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.14301361767676768, "avg_duration_per_task_ms": 429371.0404040404, "avg_output_tokens_per_task": 43957.30757575758, "last_run_timestamp": "2026-10-24T16:26:47.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8550657272727274, "accuracy_std_dev": 0.012107062717806176, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.02919473599632691, "avg_duration_per_task_ms": 757966.051882461, "avg_output_tokens_per_task": 33302.242883379244, "last_run_timestamp": "2026-10-17T08:19:33.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8549618244274809, "accuracy_std_dev": 0.005347999999999964, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.22707390839694658, "avg_duration_per_task_ms": 373958.2061068702, "avg_output_tokens_per_task": 50795.681933842236, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.8543647577679137, "accuracy_std_dev": 0.08472615418364036, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.11778292010145847, "avg_duration_per_task_ms": 1596925.525681674, "avg_output_tokens_per_task": 29301.902980342424, "last_run_timestamp": "2026-10-24T16:28:09.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8539973475609757, "accuracy_std_dev": 0.0065049450612591365, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.054438141260162595, "avg_duration_per_task_ms": 424693.43699186994, "avg_output_tokens_per_task": 27839.288617886177, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8536708232911394, "accuracy_std_dev": 0.030648058110751494, "run_count": 10, "total_questions": 1975, "avg_cost_per_task_usd": 0.008188765063291139, "avg_duration_per_task_ms": 458493.6015189873, "avg_output_tokens_per_task": 28024.251139240507, "last_run_timestamp": "2026-10-24T08:43:03.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.853535, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.028549383838383836, "avg_duration_per_task_ms": 582750.7070707071, "avg_output_tokens_per_task": 33383.742424242424, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8530642646239555, "accuracy_std_dev": 0.022329477337983097, "run_count": 4, "total_questions": 718, "avg_cost_per_task_usd": 0.02860370194986072, "avg_duration_per_task_ms": 460751.2409470752, "avg_output_tokens_per_task": 34791.85933147632, "last_run_timestamp": "2026-10-10T08:22:41.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8530032199766809, "accuracy_std_dev": 0.035466565814003675, "run_count": 39, "total_questions": 7719, "avg_cost_per_task_usd": 0.1653298276978883, "avg_duration_per_task_ms": 151135.82432957637, "avg_output_tokens_per_task": 15282.314807617568, "last_run_timestamp": "2026-10-24T08:32:13.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.8528510680564071, "accuracy_std_dev": 0.0820596725269595, "run_count": 17, "total_questions": 3262, "avg_cost_per_task_usd": 0.2673039751686082, "avg_duration_per_task_ms": 520792.21551195584, "avg_output_tokens_per_task": 67728.07817290006, "last_run_timestamp": "2026-10-24T09:05:17.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8528355426944971, "accuracy_std_dev": 0.013503895836017096, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.0925430898165718, "avg_duration_per_task_ms": 296081.5515496521, "avg_output_tokens_per_task": 29809.33523086654, "last_run_timestamp": "2026-10-24T08:21:56.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.8524375810276679, "accuracy_std_dev": 0.011217700923094702, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.07750415546772069, "avg_duration_per_task_ms": 263910.441370224, "avg_output_tokens_per_task": 26225.246376811596, "last_run_timestamp": "2026-10-24T08:40:56.292Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8524129999999999, "accuracy_std_dev": 0.012672927610549278, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1049078827160494, "avg_duration_per_task_ms": 476715.49887766555, "avg_output_tokens_per_task": 30743.156565656565, "last_run_timestamp": "2026-10-24T08:15:15.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.8521062280219781, "accuracy_std_dev": 0.006055500000000047, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.005501717032967033, "avg_duration_per_task_ms": 539158.467032967, "avg_output_tokens_per_task": 15491.851648351649, "last_run_timestamp": "2026-10-17T08:17:58.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.851852, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01747031565656566, "avg_duration_per_task_ms": 122982.43434343435, "avg_output_tokens_per_task": 28903.234848484848, "last_run_timestamp": "2026-10-24T08:15:51.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8518518181818182, "accuracy_std_dev": 0.03385326940520644, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7072226216712579, "avg_duration_per_task_ms": 481789.99357208447, "avg_output_tokens_per_task": 28288.861799816346, "last_run_timestamp": "2026-10-24T08:43:42.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8516648888888888, "accuracy_std_dev": 0.02715089739644159, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07440295791245792, "avg_duration_per_task_ms": 482923.6335578002, "avg_output_tokens_per_task": 22694.606621773288, "last_run_timestamp": "2026-10-24T16:15:14.403Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8515965972461275, "accuracy_std_dev": 0.006985364039578179, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.09721534480780264, "avg_duration_per_task_ms": 672514.3052208836, "avg_output_tokens_per_task": 40113.75157773953, "last_run_timestamp": "2026-10-24T16:04:55.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8511634577702702, "accuracy_std_dev": 0.017198169660005133, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.03812671227477478, "avg_duration_per_task_ms": 606913.3012387387, "avg_output_tokens_per_task": 35540.335585585584, "last_run_timestamp": "2026-10-24T16:36:57.539Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8509267650397275, "accuracy_std_dev": 0.029137382315888, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.14751040351872874, "avg_duration_per_task_ms": 406196.22701475595, "avg_output_tokens_per_task": 33481.99659477866, "last_run_timestamp": "2026-10-24T08:12:02.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8508249284795883, "accuracy_std_dev": 0.01638111560444773, "run_count": 33, "total_questions": 6222, "avg_cost_per_task_usd": 0.1628150337512054, "avg_duration_per_task_ms": 188242.73111539698, "avg_output_tokens_per_task": 29107.008357441337, "last_run_timestamp": "2026-10-24T09:28:03.268Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8507279006896552, "accuracy_std_dev": 0.012037642028375229, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.03107361793103448, "avg_duration_per_task_ms": 729282.3756321839, "avg_output_tokens_per_task": 32410.26482758621, "last_run_timestamp": "2026-10-17T08:19:39.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.850575, "accuracy_std_dev": null, "run_count": 1, "total_questions": 145, "avg_cost_per_task_usd": 0.015425489655172412, "avg_duration_per_task_ms": 62324.744827586204, "avg_output_tokens_per_task": 15913.096551724138, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8504104417831005, "accuracy_std_dev": 0.011757939662675208, "run_count": 9, "total_questions": 1503, "avg_cost_per_task_usd": 0.006495053226879575, "avg_duration_per_task_ms": 421761.22222222225, "avg_output_tokens_per_task": 16002.619427811045, "last_run_timestamp": "2026-10-24T16:11:33.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8504087142857142, "accuracy_std_dev": 0.010549160084565759, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.012687737373737374, "avg_duration_per_task_ms": 687002.3088023087, "avg_output_tokens_per_task": 18864.385281385283, "last_run_timestamp": "2026-10-17T16:05:11.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8502384166666668, "accuracy_std_dev": 0.009974367911955898, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09800930387205385, "avg_duration_per_task_ms": 567891.4086700337, "avg_output_tokens_per_task": 30514.193602693602, "last_run_timestamp": "2026-10-24T16:13:46.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8501683333333333, "accuracy_std_dev": 0.009523284669109108, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03896970159932659, "avg_duration_per_task_ms": 390855.180976431, "avg_output_tokens_per_task": 39310.545454545456, "last_run_timestamp": "2026-10-24T16:23:26.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8501682222222222, "accuracy_std_dev": 0.03266346068886207, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.01065054096520763, "avg_duration_per_task_ms": 397288.3883277217, "avg_output_tokens_per_task": 37715.82323232323, "last_run_timestamp": "2026-10-24T16:02:06.909Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8499550449236298, "accuracy_std_dev": 0.04367521121615224, "run_count": 7, "total_questions": 1113, "avg_cost_per_task_usd": 0.023873053908355795, "avg_duration_per_task_ms": 467262.93441150046, "avg_output_tokens_per_task": 37562.04132973944, "last_run_timestamp": "2026-10-24T08:33:50.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8497263442622951, "accuracy_std_dev": 0.01704, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.06129340163934426, "avg_duration_per_task_ms": 530265.6393442623, "avg_output_tokens_per_task": 25778.054644808744, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8497037279483036, "accuracy_std_dev": 0.02211279185187559, "run_count": 16, "total_questions": 3095, "avg_cost_per_task_usd": 0.2026981760904685, "avg_duration_per_task_ms": 259750.08206785138, "avg_output_tokens_per_task": 43394.09111470113, "last_run_timestamp": "2026-10-24T16:29:51.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8492744342105264, "accuracy_std_dev": 0.016818664646457527, "run_count": 10, "total_questions": 1976, "avg_cost_per_task_usd": 0.0980292940283401, "avg_duration_per_task_ms": 324762.6386639676, "avg_output_tokens_per_task": 28156.75556680162, "last_run_timestamp": "2026-10-24T08:53:21.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.8492166318537859, "accuracy_std_dev": 0.008781432268570992, "run_count": 4, "total_questions": 766, "avg_cost_per_task_usd": 0.1529455953002611, "avg_duration_per_task_ms": 288907.9503916449, "avg_output_tokens_per_task": 22996.231070496084, "last_run_timestamp": "2026-10-24T08:27:23.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8489659064484613, "accuracy_std_dev": 0.09543669529109948, "run_count": 21, "total_questions": 4094, "avg_cost_per_task_usd": 0.00791240718124084, "avg_duration_per_task_ms": 580031.3243771372, "avg_output_tokens_per_task": 27584.65388373229, "last_run_timestamp": "2026-10-24T08:58:59.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8489380769230769, "accuracy_std_dev": 0.01772014759645744, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.124409175990676, "avg_duration_per_task_ms": 527066.526029526, "avg_output_tokens_per_task": 46755.25407925408, "last_run_timestamp": "2026-10-24T16:02:47.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8485019556313994, "accuracy_std_dev": 0.018353041582618763, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.03159626450511945, "avg_duration_per_task_ms": 544808.5153583618, "avg_output_tokens_per_task": 33468.67861205916, "last_run_timestamp": "2026-10-24T16:10:43.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8483339713261648, "accuracy_std_dev": 0.009010597309269926, "run_count": 13, "total_questions": 2511, "avg_cost_per_task_usd": 0.06720260175228991, "avg_duration_per_task_ms": 196650.21306252488, "avg_output_tokens_per_task": 20902.762246117083, "last_run_timestamp": "2026-10-24T16:04:55.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8482398753606462, "accuracy_std_dev": 0.025739734220236125, "run_count": 9, "total_questions": 1733, "avg_cost_per_task_usd": 0.10720182342758222, "avg_duration_per_task_ms": 497095.3819965378, "avg_output_tokens_per_task": 34429.2602423543, "last_run_timestamp": "2026-10-24T08:11:11.432Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8481782975495916, "accuracy_std_dev": 0.01581318192794829, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.05179545507584597, "avg_duration_per_task_ms": 276145.2582652664, "avg_output_tokens_per_task": 45543.73667833528, "last_run_timestamp": "2026-10-24T16:18:16.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.847785277027027, "accuracy_std_dev": 0.02236998705607335, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.046796347972972975, "avg_duration_per_task_ms": 620353.6756756756, "avg_output_tokens_per_task": 38010.02759009009, "last_run_timestamp": "2026-10-24T16:27:04.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8477753864168618, "accuracy_std_dev": 0.03714670591300797, "run_count": 4, "total_questions": 427, "avg_cost_per_task_usd": 0.10432981733021078, "avg_duration_per_task_ms": 293903.51990632317, "avg_output_tokens_per_task": 24012.651053864167, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8476576319514662, "accuracy_std_dev": 0.03635809655963856, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.09557547826086957, "avg_duration_per_task_ms": 1305294.970677452, "avg_output_tokens_per_task": 42689.957532861474, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8476458282485877, "accuracy_std_dev": 0.010161445779070402, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.007148605084745762, "avg_duration_per_task_ms": 569159.6271186441, "avg_output_tokens_per_task": 17684.37118644068, "last_run_timestamp": "2026-10-24T16:26:58.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8476430833333334, "accuracy_std_dev": 0.017202829546610125, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12729099452861953, "avg_duration_per_task_ms": 386377.8813131313, "avg_output_tokens_per_task": 38419.52356902357, "last_run_timestamp": "2026-10-24T16:18:17.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8474003453908985, "accuracy_std_dev": 0.04393970502657573, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.16651004745235318, "avg_duration_per_task_ms": 164543.2010890704, "avg_output_tokens_per_task": 16431.475690392843, "last_run_timestamp": "2026-10-24T08:13:41.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8463488417849898, "accuracy_std_dev": 0.018032907906380492, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.010114639452332656, "avg_duration_per_task_ms": 449682.4756592292, "avg_output_tokens_per_task": 32532.80730223124, "last_run_timestamp": "2026-10-24T08:45:13.862Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8462835489864864, "accuracy_std_dev": 0.018064668045116323, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.06724871058558558, "avg_duration_per_task_ms": 247566.1204954955, "avg_output_tokens_per_task": 19807.05518018018, "last_run_timestamp": "2026-10-24T08:41:57.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.84624, "accuracy_std_dev": 0.02002973311521316, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.043265813131313134, "avg_duration_per_task_ms": 121558.87037037036, "avg_output_tokens_per_task": 34452.29797979798, "last_run_timestamp": "2026-10-24T08:13:28.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8459638564102563, "accuracy_std_dev": 0.012789089286005938, "run_count": 9, "total_questions": 1755, "avg_cost_per_task_usd": 0.08740559088319089, "avg_duration_per_task_ms": 1032079.9236467236, "avg_output_tokens_per_task": 44190.14074074074, "last_run_timestamp": "2026-10-24T16:33:50.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8459595, "accuracy_std_dev": 0.021043499999999993, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021401732323232326, "avg_duration_per_task_ms": 459813.37626262626, "avg_output_tokens_per_task": 42563.60858585859, "last_run_timestamp": "2026-10-24T08:30:01.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8458658888888888, "accuracy_std_dev": 0.043929875719895144, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.003920089225589225, "avg_duration_per_task_ms": 538039.7317620651, "avg_output_tokens_per_task": 39775.50785634119, "last_run_timestamp": "2026-10-24T16:14:40.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8456602006079027, "accuracy_std_dev": 0.016336342615469363, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.020697631205673757, "avg_duration_per_task_ms": 324636.22441742657, "avg_output_tokens_per_task": 41178.20871327254, "last_run_timestamp": "2026-10-24T08:56:29.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8455742847457628, "accuracy_std_dev": 0.019722621271080296, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.10396145480225989, "avg_duration_per_task_ms": 199057.26101694914, "avg_output_tokens_per_task": 30600.066666666666, "last_run_timestamp": "2026-10-24T08:23:31.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8455386666666667, "accuracy_std_dev": 0.01527369648193331, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.020861409090909087, "avg_duration_per_task_ms": 420940.80723905726, "avg_output_tokens_per_task": 41492.801346801345, "last_run_timestamp": "2026-10-24T08:13:28.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8453286747404843, "accuracy_std_dev": 0.027756094888949628, "run_count": 8, "total_questions": 1445, "avg_cost_per_task_usd": 0.09002073010380623, "avg_duration_per_task_ms": 1465880.0809688582, "avg_output_tokens_per_task": 41371.96539792387, "last_run_timestamp": "2026-10-24T16:07:10.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8450579454306376, "accuracy_std_dev": 0.010092084710428701, "run_count": 9, "total_questions": 1521, "avg_cost_per_task_usd": 0.13361581196581196, "avg_duration_per_task_ms": 130442.2596975674, "avg_output_tokens_per_task": 31421.66929651545, "last_run_timestamp": "2026-10-24T08:11:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8449909073783358, "accuracy_std_dev": 0.02024798218992146, "run_count": 29, "total_questions": 5733, "avg_cost_per_task_usd": 0.24276903209488926, "avg_duration_per_task_ms": 502111.777603349, "avg_output_tokens_per_task": 47287.581719867434, "last_run_timestamp": "2026-10-24T16:21:25.073Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8449648181818183, "accuracy_std_dev": 0.05078720577757961, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.16265579063360883, "avg_duration_per_task_ms": 162118.34848484848, "avg_output_tokens_per_task": 16092.63269054178, "last_run_timestamp": "2026-10-24T08:18:32.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.844913105281815, "accuracy_std_dev": 0.019516578349245098, "run_count": 15, "total_questions": 2821, "avg_cost_per_task_usd": 0.13749859695143568, "avg_duration_per_task_ms": 278207.19319390284, "avg_output_tokens_per_task": 28367.055654023396, "last_run_timestamp": "2026-10-24T08:44:43.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8448157507598784, "accuracy_std_dev": 0.02762329063670728, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.013124052684903749, "avg_duration_per_task_ms": 664201.0739614995, "avg_output_tokens_per_task": 40826.44782168186, "last_run_timestamp": "2026-10-24T08:41:23.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8444763809523811, "accuracy_std_dev": 0.010977878186331046, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.14939990764790767, "avg_duration_per_task_ms": 163824.38455988455, "avg_output_tokens_per_task": 32857.31721981722, "last_run_timestamp": "2026-10-24T08:45:58.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.844398036338225, "accuracy_std_dev": 0.011789574159373825, "run_count": 8, "total_questions": 1431, "avg_cost_per_task_usd": 0.015440737246680642, "avg_duration_per_task_ms": 650403.249475891, "avg_output_tokens_per_task": 16520.188679245282, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8443693333333332, "accuracy_std_dev": 0.01866431008338881, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1411051307519641, "avg_duration_per_task_ms": 894934.3092031425, "avg_output_tokens_per_task": 55451.94893378227, "last_run_timestamp": "2026-10-24T16:03:25.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8441825555555555, "accuracy_std_dev": 0.02302992819070732, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.014299253647586982, "avg_duration_per_task_ms": 527298.7811447812, "avg_output_tokens_per_task": 38122.5847362514, "last_run_timestamp": "2026-10-24T08:42:05.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8440474857142857, "accuracy_std_dev": 0.0037822104436426945, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.11554378775510205, "avg_duration_per_task_ms": 243142.18265306123, "avg_output_tokens_per_task": 31758.279591836734, "last_run_timestamp": "2026-10-24T08:42:05.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8440043168316831, "accuracy_std_dev": 0.006860781750965057, "run_count": 8, "total_questions": 1515, "avg_cost_per_task_usd": 0.05332531353135314, "avg_duration_per_task_ms": 266581.80264026404, "avg_output_tokens_per_task": 22128.052145214522, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.843855125, "accuracy_std_dev": 0.007974456116838023, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1439108863636364, "avg_duration_per_task_ms": 148669.27525252526, "avg_output_tokens_per_task": 32529.41856060606, "last_run_timestamp": "2026-10-24T08:10:23.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.8438239857868021, "accuracy_std_dev": 0.012941643289783599, "run_count": 5, "total_questions": 985, "avg_cost_per_task_usd": 0.272832538071066, "avg_duration_per_task_ms": 511204.8578680203, "avg_output_tokens_per_task": 61801.45989847716, "last_run_timestamp": "2026-10-24T16:14:04.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8434343846153846, "accuracy_std_dev": 0.032261314395795763, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.0527346484071484, "avg_duration_per_task_ms": 208837.4222999223, "avg_output_tokens_per_task": 21871.209401709402, "last_run_timestamp": "2026-10-24T16:02:47.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8427474515050167, "accuracy_std_dev": 0.014940270952109593, "run_count": 14, "total_questions": 2691, "avg_cost_per_task_usd": 0.021943730955035302, "avg_duration_per_task_ms": 258715.1181716834, "avg_output_tokens_per_task": 37901.94314381271, "last_run_timestamp": "2026-10-24T08:49:23.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8423346063291139, "accuracy_std_dev": 0.054296645253950386, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.07867724514767932, "avg_duration_per_task_ms": 564357.3624472574, "avg_output_tokens_per_task": 43556.894092827, "last_run_timestamp": "2026-10-24T16:27:11.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8419978532110092, "accuracy_std_dev": 0.010540078834619807, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.027336418960244645, "avg_duration_per_task_ms": 516242.43323139654, "avg_output_tokens_per_task": 22594.290519877675, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8419939695817491, "accuracy_std_dev": 0.05773343437894112, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.08744185044359949, "avg_duration_per_task_ms": 697734.2648922687, "avg_output_tokens_per_task": 26121.35234474018, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.84196125, "accuracy_std_dev": 0.04164286996518729, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10050041856060607, "avg_duration_per_task_ms": 1121158.3566919193, "avg_output_tokens_per_task": 39757.59027777778, "last_run_timestamp": "2026-10-24T16:02:28.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8415218367346938, "accuracy_std_dev": 0.019781995922842507, "run_count": 9, "total_questions": 1323, "avg_cost_per_task_usd": 0.017032966742252457, "avg_duration_per_task_ms": 90861.69312169313, "avg_output_tokens_per_task": 14612.785336356765, "last_run_timestamp": "2026-10-24T16:27:17.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8415046100309946, "accuracy_std_dev": 0.012823272459877284, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.11274710115525499, "avg_duration_per_task_ms": 322682.1735700197, "avg_output_tokens_per_task": 34551.41730064807, "last_run_timestamp": "2026-10-24T08:41:42.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.84141817433047, "accuracy_std_dev": 0.011968232237469333, "run_count": 10, "total_questions": 1979, "avg_cost_per_task_usd": 0.14348533804951996, "avg_duration_per_task_ms": 205361.71248105104, "avg_output_tokens_per_task": 32378.852956038405, "last_run_timestamp": "2026-10-24T08:12:37.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8413977096774194, "accuracy_std_dev": 0.007842951293996439, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.03688066532258065, "avg_duration_per_task_ms": 47685.283602150535, "avg_output_tokens_per_task": 14664.099462365592, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8411896666666667, "accuracy_std_dev": 0.008729468916008341, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.036658867003367, "avg_duration_per_task_ms": 51092.77104377104, "avg_output_tokens_per_task": 14574.808080808081, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8409475076142132, "accuracy_std_dev": 0.005443376341940744, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.020375204737732657, "avg_duration_per_task_ms": 653417.3384094755, "avg_output_tokens_per_task": 23451.566835871403, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8408155555555555, "accuracy_std_dev": 0.0506301697894998, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09469512233445566, "avg_duration_per_task_ms": 391927.05611672276, "avg_output_tokens_per_task": 21250.56509539843, "last_run_timestamp": "2026-10-24T16:06:30.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.84048825, "accuracy_std_dev": 0.004667830993887862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06197197222222223, "avg_duration_per_task_ms": 84839.70707070707, "avg_output_tokens_per_task": 13643.420454545454, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8402775531250001, "accuracy_std_dev": 0.007466846475587956, "run_count": 10, "total_questions": 1920, "avg_cost_per_task_usd": 0.039408469791666664, "avg_duration_per_task_ms": 51791.722395833334, "avg_output_tokens_per_task": 14757.431770833333, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8402425498181818, "accuracy_std_dev": 0.00976753775827742, "run_count": 8, "total_questions": 1375, "avg_cost_per_task_usd": 0.0796483170909091, "avg_duration_per_task_ms": 509723.63709090906, "avg_output_tokens_per_task": 19683.845818181817, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8398882325043376, "accuracy_std_dev": 0.015516849481773015, "run_count": 10, "total_questions": 1729, "avg_cost_per_task_usd": 0.14290184152689414, "avg_duration_per_task_ms": 273124.987854251, "avg_output_tokens_per_task": 33153.25390399075, "last_run_timestamp": "2026-10-24T08:53:46.038Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.839837363066202, "accuracy_std_dev": 0.014816504049501316, "run_count": 15, "total_questions": 2870, "avg_cost_per_task_usd": 0.07680571289198605, "avg_duration_per_task_ms": 389476.49477351917, "avg_output_tokens_per_task": 22208.951219512193, "last_run_timestamp": "2026-10-24T08:10:23.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.839646375, "accuracy_std_dev": 0.01029199634348825, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006103154040404041, "avg_duration_per_task_ms": 431518.17424242425, "avg_output_tokens_per_task": 25837.31818181818, "last_run_timestamp": "2026-10-24T16:03:25.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8396166498586238, "accuracy_std_dev": 0.03493831082667691, "run_count": 11, "total_questions": 2122, "avg_cost_per_task_usd": 0.029913213949104622, "avg_duration_per_task_ms": 288209.38784165884, "avg_output_tokens_per_task": 9239.83930254477, "last_run_timestamp": "2026-10-24T08:08:54.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8389520859671303, "accuracy_std_dev": 0.014418680686632029, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.22912267003792672, "avg_duration_per_task_ms": 279595.97260851244, "avg_output_tokens_per_task": 51927.226717235564, "last_run_timestamp": "2026-10-24T16:04:55.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.8389448333333334, "accuracy_std_dev": 0.03599784248820413, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004737409090909091, "avg_duration_per_task_ms": 1034495.3484848485, "avg_output_tokens_per_task": 27462.325757575756, "last_run_timestamp": "2026-10-24T16:19:29.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.838522452928095, "accuracy_std_dev": 0.010501612316687394, "run_count": 8, "total_questions": 1349, "avg_cost_per_task_usd": 0.006855791697553743, "avg_duration_per_task_ms": 622066.7916975537, "avg_output_tokens_per_task": 16928.25722757598, "last_run_timestamp": "2026-10-24T16:21:33.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.838520908319185, "accuracy_std_dev": 0.021628456230475167, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.09386730616864743, "avg_duration_per_task_ms": 1055070.8788907754, "avg_output_tokens_per_task": 42969.125636672325, "last_run_timestamp": "2026-10-24T16:12:09.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.8383968759493671, "accuracy_std_dev": 0.006763999999999992, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.033454222784810125, "avg_duration_per_task_ms": 81357.47848101266, "avg_output_tokens_per_task": 14534.177215189873, "last_run_timestamp": "2026-10-24T09:12:44.274Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.83813855538695, "accuracy_std_dev": 0.02473346097415401, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.1589729180576631, "avg_duration_per_task_ms": 941860.1360647446, "avg_output_tokens_per_task": 52231.05108750632, "last_run_timestamp": "2026-10-24T16:05:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8378745924484235, "accuracy_std_dev": 0.026005006584645496, "run_count": 13, "total_questions": 2569, "avg_cost_per_task_usd": 0.08716310782405605, "avg_duration_per_task_ms": 387520.7785130401, "avg_output_tokens_per_task": 21670.57571039315, "last_run_timestamp": "2026-10-24T16:33:45.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8378226666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.15828378956228956, "avg_duration_per_task_ms": 446608.7861952862, "avg_output_tokens_per_task": 56298.93097643097, "last_run_timestamp": "2026-10-10T08:30:28.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8377103, "accuracy_std_dev": 0.023183391999662194, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10627496717171718, "avg_duration_per_task_ms": 386419.75505050505, "avg_output_tokens_per_task": 35856.45909090909, "last_run_timestamp": "2026-10-24T08:25:29.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.837692958038147, "accuracy_std_dev": 0.020571358270420545, "run_count": 10, "total_questions": 1835, "avg_cost_per_task_usd": 0.07314428610354223, "avg_duration_per_task_ms": 380364.9722070845, "avg_output_tokens_per_task": 24128.798910081743, "last_run_timestamp": "2026-10-24T16:08:29.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8376859309031556, "accuracy_std_dev": 0.027367300407603226, "run_count": 10, "total_questions": 1838, "avg_cost_per_task_usd": 0.011182803590859632, "avg_duration_per_task_ms": 390045.14526659413, "avg_output_tokens_per_task": 22148.7992383025, "last_run_timestamp": "2026-10-24T08:39:40.816Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8375419999999999, "accuracy_std_dev": 0.012922002863333565, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.15197074579124578, "avg_duration_per_task_ms": 226641.9739057239, "avg_output_tokens_per_task": 34291.69781144781, "last_run_timestamp": "2026-10-24T08:43:12.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.83744096, "accuracy_std_dev": 0.012056350192259705, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.2619175503030303, "avg_duration_per_task_ms": 272403.29393939395, "avg_output_tokens_per_task": 56992.821616161615, "last_run_timestamp": "2026-10-24T16:27:11.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8372897388724037, "accuracy_std_dev": 0.028847852416095617, "run_count": 12, "total_questions": 2359, "avg_cost_per_task_usd": 0.06617999915218314, "avg_duration_per_task_ms": 1053779.2886816447, "avg_output_tokens_per_task": 33011.97880457821, "last_run_timestamp": "2026-10-24T16:02:06.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8371210833333332, "accuracy_std_dev": 0.012487611784340083, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0845222638888889, "avg_duration_per_task_ms": 727103.742003367, "avg_output_tokens_per_task": 48945.89772727273, "last_run_timestamp": "2026-10-24T16:11:13.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8368686, "accuracy_std_dev": 0.01362896250783603, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1397023909090909, "avg_duration_per_task_ms": 182914.46161616163, "avg_output_tokens_per_task": 31522.08787878788, "last_run_timestamp": "2026-10-24T08:24:39.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8367790607594936, "accuracy_std_dev": 0.029638370939918782, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.017527371729957806, "avg_duration_per_task_ms": 145536.47299578058, "avg_output_tokens_per_task": 18246.05611814346, "last_run_timestamp": "2026-10-24T16:35:32.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.8367000000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.18573769191919193, "avg_duration_per_task_ms": 281534.297979798, "avg_output_tokens_per_task": 34913.7095959596, "last_run_timestamp": "2026-10-24T08:56:15.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8366667090476191, "accuracy_std_dev": 0.01978861195373632, "run_count": 12, "total_questions": 2100, "avg_cost_per_task_usd": 0.09574838047619047, "avg_duration_per_task_ms": 613293.1757142857, "avg_output_tokens_per_task": 29845.292857142857, "last_run_timestamp": "2026-10-24T16:10:16.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.836653454183267, "accuracy_std_dev": 0.007833999999999952, "run_count": 2, "total_questions": 251, "avg_cost_per_task_usd": 0.06977503187250997, "avg_duration_per_task_ms": 237256.35059760956, "avg_output_tokens_per_task": 17511.450199203187, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8360894826189674, "accuracy_std_dev": 0.027487401181389745, "run_count": 15, "total_questions": 2963, "avg_cost_per_task_usd": 0.02392725244684441, "avg_duration_per_task_ms": 312624.6992912589, "avg_output_tokens_per_task": 17783.592642591968, "last_run_timestamp": "2026-10-24T08:22:23.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.83599875, "accuracy_std_dev": 0.06733100699544627, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.20365952861952863, "avg_duration_per_task_ms": 537295.401936027, "avg_output_tokens_per_task": 46280.556818181816, "last_run_timestamp": "2026-10-24T16:48:27.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.835384616923077, "accuracy_std_dev": 0.032642666915557006, "run_count": 5, "total_questions": 650, "avg_cost_per_task_usd": 0.09615046153846153, "avg_duration_per_task_ms": 334242.9476923077, "avg_output_tokens_per_task": 39819.60307692308, "last_run_timestamp": "2026-10-24T16:40:24.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8353175277777779, "accuracy_std_dev": 0.01836981616741691, "run_count": 9, "total_questions": 1764, "avg_cost_per_task_usd": 0.014329623015873016, "avg_duration_per_task_ms": 445894.7193877551, "avg_output_tokens_per_task": 39122.69671201814, "last_run_timestamp": "2026-10-24T08:17:53.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8350166666666667, "accuracy_std_dev": 0.005991526757754575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06194655555555556, "avg_duration_per_task_ms": 84396.87373737374, "avg_output_tokens_per_task": 13638.5, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.834789191724138, "accuracy_std_dev": 0.03295145245372225, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.10863842022988504, "avg_duration_per_task_ms": 1052937.216091954, "avg_output_tokens_per_task": 33937.42436781609, "last_run_timestamp": "2026-10-24T16:17:57.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.834628230769231, "accuracy_std_dev": 0.032424071580691764, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.05403845415695415, "avg_duration_per_task_ms": 253079.37917637918, "avg_output_tokens_per_task": 22985.918026418025, "last_run_timestamp": "2026-10-24T16:06:30.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8346143259031515, "accuracy_std_dev": 0.012531384063964916, "run_count": 16, "total_questions": 2602, "avg_cost_per_task_usd": 0.1179007190622598, "avg_duration_per_task_ms": 273695.7221368178, "avg_output_tokens_per_task": 27899.308224442735, "last_run_timestamp": "2026-10-24T08:12:02.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8340433944004868, "accuracy_std_dev": 0.008747099804372552, "run_count": 9, "total_questions": 1643, "avg_cost_per_task_usd": 0.04404786731588557, "avg_duration_per_task_ms": 94664.89896530737, "avg_output_tokens_per_task": 15175.068167985393, "last_run_timestamp": "2026-10-24T08:10:23.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8339663204791117, "accuracy_std_dev": 0.017426212050589154, "run_count": 18, "total_questions": 3423, "avg_cost_per_task_usd": 0.25302901986561493, "avg_duration_per_task_ms": 374119.30879345606, "avg_output_tokens_per_task": 43927.57055214724, "last_run_timestamp": "2026-10-24T16:03:06.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8337216923076924, "accuracy_std_dev": 0.011740948902712052, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.2507070532245532, "avg_duration_per_task_ms": 380155.1087801088, "avg_output_tokens_per_task": 40506.53496503497, "last_run_timestamp": "2026-10-24T08:43:07.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8336574603112842, "accuracy_std_dev": 0.021426026057470033, "run_count": 13, "total_questions": 2570, "avg_cost_per_task_usd": 0.024801823735408558, "avg_duration_per_task_ms": 393313.6081712062, "avg_output_tokens_per_task": 46122.12840466926, "last_run_timestamp": "2026-10-24T08:48:13.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8334863636363636, "accuracy_std_dev": 0.011783391626836715, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0389523406795225, "avg_duration_per_task_ms": 226530.9031221304, "avg_output_tokens_per_task": 14515.32966023875, "last_run_timestamp": "2026-10-24T08:08:52.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8328383188836104, "accuracy_std_dev": 0.012861707810491917, "run_count": 9, "total_questions": 1684, "avg_cost_per_task_usd": 0.2188719726840855, "avg_duration_per_task_ms": 450599.8604513064, "avg_output_tokens_per_task": 49626.254156769595, "last_run_timestamp": "2026-10-24T16:08:01.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8327628225898459, "accuracy_std_dev": 0.014159242972544667, "run_count": 9, "total_questions": 1753, "avg_cost_per_task_usd": 0.15039315516257845, "avg_duration_per_task_ms": 277652.9520821449, "avg_output_tokens_per_task": 33995.16029663434, "last_run_timestamp": "2026-10-24T08:12:02.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.832178375487224, "accuracy_std_dev": 0.011051436981414083, "run_count": 12, "total_questions": 2309, "avg_cost_per_task_usd": 0.2290666569943699, "avg_duration_per_task_ms": 296861.6738847986, "avg_output_tokens_per_task": 51892.169770463406, "last_run_timestamp": "2026-10-24T16:31:42.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8320383846153845, "accuracy_std_dev": 0.008064443483478893, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.07931912276612277, "avg_duration_per_task_ms": 701901.0648795649, "avg_output_tokens_per_task": 57457.6013986014, "last_run_timestamp": "2026-10-24T16:24:24.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8319781768292682, "accuracy_std_dev": 0.02156774450887252, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.07742283333333333, "avg_duration_per_task_ms": 694188.4644308944, "avg_output_tokens_per_task": 41828.17479674797, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.83176317025572, "accuracy_std_dev": 0.014297695836545667, "run_count": 8, "total_questions": 1486, "avg_cost_per_task_usd": 0.023221829744279946, "avg_duration_per_task_ms": 299260.8001345895, "avg_output_tokens_per_task": 46221.84320323015, "last_run_timestamp": "2026-10-24T08:22:23.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.83165, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009051732323232323, "avg_duration_per_task_ms": 428104.83838383836, "avg_output_tokens_per_task": 21594.323232323233, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8313810512445096, "accuracy_std_dev": 0.012942947508356888, "run_count": 7, "total_questions": 1366, "avg_cost_per_task_usd": 0.027591674963396776, "avg_duration_per_task_ms": 601596.977306003, "avg_output_tokens_per_task": 14650.56149341142, "last_run_timestamp": "2026-10-24T08:48:04.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.831313, "accuracy_std_dev": 0.01944119231940264, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025460874747474747, "avg_duration_per_task_ms": 242198.30808080808, "avg_output_tokens_per_task": 21221.607070707072, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.83122875, "accuracy_std_dev": 0.007839878231675557, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022249369949494953, "avg_duration_per_task_ms": 11972.136363636364, "avg_output_tokens_per_task": 14384.287878787878, "last_run_timestamp": "2026-10-24T08:21:05.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8311627944186045, "accuracy_std_dev": 0.006786821620057476, "run_count": 11, "total_questions": 2150, "avg_cost_per_task_usd": 0.03491511023255814, "avg_duration_per_task_ms": 201279.6446511628, "avg_output_tokens_per_task": 14261.299069767441, "last_run_timestamp": "2026-10-24T08:41:23.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8310956896816264, "accuracy_std_dev": 0.010867420886961873, "run_count": 16, "total_questions": 2607, "avg_cost_per_task_usd": 0.020108517453011125, "avg_duration_per_task_ms": 239467.13693901035, "avg_output_tokens_per_task": 40022.99923283468, "last_run_timestamp": "2026-10-24T08:13:46.749Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8310885000000001, "accuracy_std_dev": 0.012955632388399007, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06418621717171719, "avg_duration_per_task_ms": 117116.24831649831, "avg_output_tokens_per_task": 13483.090067340067, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.830874032193159, "accuracy_std_dev": 0.029241810305408605, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.00674413145539906, "avg_duration_per_task_ms": 717174.3158953722, "avg_output_tokens_per_task": 18114.10865191147, "last_run_timestamp": "2026-10-24T16:05:36.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8306676666666667, "accuracy_std_dev": 0.02403995285052688, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.13645198905723904, "avg_duration_per_task_ms": 551462.0761784512, "avg_output_tokens_per_task": 43445.10816498317, "last_run_timestamp": "2026-10-24T16:02:47.376Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8302065714285713, "accuracy_std_dev": 0.0046260801628883914, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.05203393939393939, "avg_duration_per_task_ms": 277192.08946608944, "avg_output_tokens_per_task": 15155.578643578643, "last_run_timestamp": "2026-10-24T08:29:16.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8301606153846154, "accuracy_std_dev": 0.025057780433164603, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.038417987567987574, "avg_duration_per_task_ms": 349233.05205905205, "avg_output_tokens_per_task": 39432.61577311577, "last_run_timestamp": "2026-10-24T16:01:48.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8301281231808731, "accuracy_std_dev": 0.02299658002943048, "run_count": 10, "total_questions": 1924, "avg_cost_per_task_usd": 0.021267546777546775, "avg_duration_per_task_ms": 232718.9979209979, "avg_output_tokens_per_task": 42313.036902286905, "last_run_timestamp": "2026-10-24T08:44:41.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8298259166666667, "accuracy_std_dev": 0.01956609097162035, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.2451282104377105, "avg_duration_per_task_ms": 378368.86447811447, "avg_output_tokens_per_task": 55545.628367003366, "last_run_timestamp": "2026-10-24T16:24:45.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8294438755401903, "accuracy_std_dev": 0.015719092947537823, "run_count": 15, "total_questions": 2314, "avg_cost_per_task_usd": 0.038434746326706994, "avg_duration_per_task_ms": 267856.29127052723, "avg_output_tokens_per_task": 30639.18668971478, "last_run_timestamp": "2026-10-24T16:05:40.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8286576957919346, "accuracy_std_dev": 0.020416737880752586, "run_count": 18, "total_questions": 3422, "avg_cost_per_task_usd": 0.04187343191116307, "avg_duration_per_task_ms": 431990.98655756866, "avg_output_tokens_per_task": 45586.554061952076, "last_run_timestamp": "2026-10-24T16:01:48.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8284511, "accuracy_std_dev": 0.007394059013153757, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.14032680555555557, "avg_duration_per_task_ms": 177948.25353535355, "avg_output_tokens_per_task": 34732.438888888886, "last_run_timestamp": "2026-10-24T08:41:54.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.8283759292307693, "accuracy_std_dev": 0.023115878020096917, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.0696730758974359, "avg_duration_per_task_ms": 827348.3805128206, "avg_output_tokens_per_task": 39830.013333333336, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.82828275, "accuracy_std_dev": 0.004610363346581274, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0223316047979798, "avg_duration_per_task_ms": 615131.3409090909, "avg_output_tokens_per_task": 56159.15404040404, "last_run_timestamp": "2026-10-17T08:18:07.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8280900921348314, "accuracy_std_dev": 0.00897116480731462, "run_count": 5, "total_questions": 890, "avg_cost_per_task_usd": 0.02632537640449438, "avg_duration_per_task_ms": 122831.4988764045, "avg_output_tokens_per_task": 20925.239325842696, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8276692169459963, "accuracy_std_dev": 0.017541679518523267, "run_count": 11, "total_questions": 2148, "avg_cost_per_task_usd": 0.14409020577281192, "avg_duration_per_task_ms": 292895.9767225326, "avg_output_tokens_per_task": 32531.135009310987, "last_run_timestamp": "2026-10-24T08:12:02.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8273709166666667, "accuracy_std_dev": 0.056014389335030584, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.051560792508417504, "avg_duration_per_task_ms": 531412.6561447812, "avg_output_tokens_per_task": 24685.848905723906, "last_run_timestamp": "2026-10-24T16:11:29.497Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8273257868177137, "accuracy_std_dev": 0.013461365335284555, "run_count": 10, "total_questions": 1942, "avg_cost_per_task_usd": 0.0351326035015448, "avg_duration_per_task_ms": 424285.9001029866, "avg_output_tokens_per_task": 48592.330072090626, "last_run_timestamp": "2026-10-17T08:18:01.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8272187207100591, "accuracy_std_dev": 0.0496186451996, "run_count": 13, "total_questions": 2535, "avg_cost_per_task_usd": 0.10137412702169626, "avg_duration_per_task_ms": 538485.309270217, "avg_output_tokens_per_task": 29237.781459566075, "last_run_timestamp": "2026-10-24T08:19:59.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8271604444444445, "accuracy_std_dev": 0.023434861676623345, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0269165101010101, "avg_duration_per_task_ms": 568738.2138047138, "avg_output_tokens_per_task": 46021.95230078563, "last_run_timestamp": "2026-10-17T08:17:59.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8269359, "accuracy_std_dev": 0.017098920278485433, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1309031328282828, "avg_duration_per_task_ms": 311479.3813131313, "avg_output_tokens_per_task": 54002.43080808081, "last_run_timestamp": "2026-10-24T16:04:33.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.826809625, "accuracy_std_dev": 0.013025065527066464, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09912519444444444, "avg_duration_per_task_ms": 360768.5435606061, "avg_output_tokens_per_task": 45556.78724747475, "last_run_timestamp": "2026-10-17T16:06:16.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8267677, "accuracy_std_dev": 0.010999569410208776, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.03662119747474748, "avg_duration_per_task_ms": 290125.5409090909, "avg_output_tokens_per_task": 15502.71515151515, "last_run_timestamp": "2026-10-24T08:13:28.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.8265989999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009182863636363637, "avg_duration_per_task_ms": 852550.7929292929, "avg_output_tokens_per_task": 21914.838383838385, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8265763479729729, "accuracy_std_dev": 0.010997465889922129, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.00803203153153153, "avg_duration_per_task_ms": 497969.4971846847, "avg_output_tokens_per_task": 20675.66554054054, "last_run_timestamp": "2026-10-24T16:12:11.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8264309000000002, "accuracy_std_dev": 0.009654922303674958, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04312889949494949, "avg_duration_per_task_ms": 229184.7, "avg_output_tokens_per_task": 14260.256565656566, "last_run_timestamp": "2026-10-24T08:44:44.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8263402307692308, "accuracy_std_dev": 0.013738089499657219, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.06289175718725717, "avg_duration_per_task_ms": 535684.9638694639, "avg_output_tokens_per_task": 49516.1418026418, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8258501862244899, "accuracy_std_dev": 0.010007618232626584, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.007764268367346939, "avg_duration_per_task_ms": 349312.62346938776, "avg_output_tokens_per_task": 19101.2056122449, "last_run_timestamp": "2026-10-24T16:01:08.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.8257575, "accuracy_std_dev": 0.017676499999999984, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02807809090909091, "avg_duration_per_task_ms": 230919.77777777778, "avg_output_tokens_per_task": 55916.378787878784, "last_run_timestamp": "2026-10-24T08:15:10.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8254771111111111, "accuracy_std_dev": 0.00965472805813521, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.047990214365881044, "avg_duration_per_task_ms": 442130.5162738496, "avg_output_tokens_per_task": 26578.690796857463, "last_run_timestamp": "2026-10-24T16:31:34.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8253999912733042, "accuracy_std_dev": 0.022060427402287506, "run_count": 13, "total_questions": 2521, "avg_cost_per_task_usd": 0.027058457754859183, "avg_duration_per_task_ms": 873701.1269337564, "avg_output_tokens_per_task": 48650.64220547402, "last_run_timestamp": "2026-10-24T08:26:00.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8253968696428573, "accuracy_std_dev": 0.014772865576229071, "run_count": 9, "total_questions": 1680, "avg_cost_per_task_usd": 0.024977942261904765, "avg_duration_per_task_ms": 272190.5416666667, "avg_output_tokens_per_task": 49745.9630952381, "last_run_timestamp": "2026-10-24T08:27:40.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8252562970922881, "accuracy_std_dev": 0.043641090936944585, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.08621131184155077, "avg_duration_per_task_ms": 759710.0198061526, "avg_output_tokens_per_task": 31352.24863042562, "last_run_timestamp": "2026-10-24T16:13:03.130Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.8247287777777778, "accuracy_std_dev": 0.07543645506101702, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.012084622895622895, "avg_duration_per_task_ms": 775310.8434343435, "avg_output_tokens_per_task": 26356.983164983165, "last_run_timestamp": "2026-10-24T16:04:55.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8246474711042312, "accuracy_std_dev": 0.04610697461436829, "run_count": 10, "total_questions": 1938, "avg_cost_per_task_usd": 0.019926908668730647, "avg_duration_per_task_ms": 787644.0030959753, "avg_output_tokens_per_task": 45616.08255933953, "last_run_timestamp": "2026-10-24T08:22:23.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8244948000000002, "accuracy_std_dev": 0.0601779527199123, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.028505976262626263, "avg_duration_per_task_ms": 700158.9757575757, "avg_output_tokens_per_task": 29700.529292929292, "last_run_timestamp": "2026-10-17T08:18:04.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8241368042686101, "accuracy_std_dev": 0.01320757262179545, "run_count": 10, "total_questions": 1921, "avg_cost_per_task_usd": 0.024530483081728265, "avg_duration_per_task_ms": 442142.1447162936, "avg_output_tokens_per_task": 24482.630921395106, "last_run_timestamp": "2026-10-24T16:11:33.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8241029543178973, "accuracy_std_dev": 0.01722442160378035, "run_count": 17, "total_questions": 3196, "avg_cost_per_task_usd": 0.030665748122653304, "avg_duration_per_task_ms": 264011.0794743429, "avg_output_tokens_per_task": 13973.099186483103, "last_run_timestamp": "2026-10-24T08:58:05.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.8240257172774869, "accuracy_std_dev": 0.000479814547507715, "run_count": 3, "total_questions": 573, "avg_cost_per_task_usd": 0.1285148184991274, "avg_duration_per_task_ms": 295843.96684118675, "avg_output_tokens_per_task": 31965.242582897034, "last_run_timestamp": "2026-10-24T08:50:12.983Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8239975454545455, "accuracy_std_dev": 0.022862830634275574, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.010785916437098255, "avg_duration_per_task_ms": 717852.0280073462, "avg_output_tokens_per_task": 38282.08126721763, "last_run_timestamp": "2026-10-17T08:02:36.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.8239540000000001, "accuracy_std_dev": 0.009474601205327877, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14372687085137084, "avg_duration_per_task_ms": 420021.29581529583, "avg_output_tokens_per_task": 28768.17316017316, "last_run_timestamp": "2026-10-24T08:48:18.601Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8239056, "accuracy_std_dev": 0.013153034579138006, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02469372222222222, "avg_duration_per_task_ms": 472106.2909090909, "avg_output_tokens_per_task": 23003.946464646466, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.823871008431703, "accuracy_std_dev": 0.014025260116423268, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.19393739516582348, "avg_duration_per_task_ms": 941267.5373805509, "avg_output_tokens_per_task": 51620.3125351321, "last_run_timestamp": "2026-10-24T16:12:49.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8236976925031766, "accuracy_std_dev": 0.11602625064229809, "run_count": 8, "total_questions": 1574, "avg_cost_per_task_usd": 0.12381996124523506, "avg_duration_per_task_ms": 755214.6099110546, "avg_output_tokens_per_task": 38337.82973316391, "last_run_timestamp": "2026-10-24T16:03:25.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8235687999999998, "accuracy_std_dev": 0.003085731900214278, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.018559439393939392, "avg_duration_per_task_ms": 535812.6131313131, "avg_output_tokens_per_task": 18445.026262626263, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.823296396946565, "accuracy_std_dev": 0.009206241083574164, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.023398153519932147, "avg_duration_per_task_ms": 341450.7421543681, "avg_output_tokens_per_task": 20864.225614927906, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.823232125, "accuracy_std_dev": 0.02934086116850314, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09752844696969695, "avg_duration_per_task_ms": 1429250.3825757576, "avg_output_tokens_per_task": 44988.86111111111, "last_run_timestamp": "2026-10-24T16:04:55.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025912994949494946, "avg_duration_per_task_ms": 162312.898989899, "avg_output_tokens_per_task": 9036.257575757576, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8223551976047904, "accuracy_std_dev": 0.029631373343505716, "run_count": 6, "total_questions": 1169, "avg_cost_per_task_usd": 0.06724194183062446, "avg_duration_per_task_ms": 236148.74337040205, "avg_output_tokens_per_task": 20314.131736526946, "last_run_timestamp": "2026-10-17T08:26:17.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8223140000000001, "accuracy_std_dev": 0.020822165248512558, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.012249293847566574, "avg_duration_per_task_ms": 720758.7061524334, "avg_output_tokens_per_task": 36235.862258953166, "last_run_timestamp": "2026-10-17T08:04:21.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8223139999999999, "accuracy_std_dev": 0.02011718801965576, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03311863039485767, "avg_duration_per_task_ms": 262977.6432506887, "avg_output_tokens_per_task": 14545.10927456382, "last_run_timestamp": "2026-10-24T08:29:59.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.82211, "accuracy_std_dev": 0.057194952006274134, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09893642424242423, "avg_duration_per_task_ms": 769490.925925926, "avg_output_tokens_per_task": 39542.70622895623, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.821895294117647, "accuracy_std_dev": 0.07409118108715027, "run_count": 3, "total_questions": 408, "avg_cost_per_task_usd": 0.13934752450980392, "avg_duration_per_task_ms": 1455873.1495098039, "avg_output_tokens_per_task": 48235.80147058824, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.8214756245059289, "accuracy_std_dev": 0.05273064972335267, "run_count": 9, "total_questions": 1771, "avg_cost_per_task_usd": 0.18877710615471485, "avg_duration_per_task_ms": 607666.5719932242, "avg_output_tokens_per_task": 44926.648785996615, "last_run_timestamp": "2026-10-24T16:01:45.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8214198161925602, "accuracy_std_dev": 0.04954945097072217, "run_count": 8, "total_questions": 1371, "avg_cost_per_task_usd": 0.04645164551422319, "avg_duration_per_task_ms": 608936.6484318016, "avg_output_tokens_per_task": 28638.388767323122, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8212682499999999, "accuracy_std_dev": 0.0185418798135689, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.04273052441077441, "avg_duration_per_task_ms": 239740.20075757575, "avg_output_tokens_per_task": 14238.065656565657, "last_run_timestamp": "2026-10-24T08:50:39.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8210063161094224, "accuracy_std_dev": 0.031028311643400763, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.025833400202634246, "avg_duration_per_task_ms": 497222.336372847, "avg_output_tokens_per_task": 24022.691995947316, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8202537692307693, "accuracy_std_dev": 0.0131593300691875, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.31460015734265734, "avg_duration_per_task_ms": 289943.24397824396, "avg_output_tokens_per_task": 20783.278943278943, "last_run_timestamp": "2026-10-24T08:12:02.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8199838489742471, "accuracy_std_dev": 0.060002328561629244, "run_count": 24, "total_questions": 4582, "avg_cost_per_task_usd": 0.007503320384111742, "avg_duration_per_task_ms": 496893.82627673505, "avg_output_tokens_per_task": 17430.922959406373, "last_run_timestamp": "2026-10-24T16:15:47.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.8197662921174653, "accuracy_std_dev": 0.015949041727953438, "run_count": 10, "total_questions": 1941, "avg_cost_per_task_usd": 0.018771416280267902, "avg_duration_per_task_ms": 575656.4595569294, "avg_output_tokens_per_task": 20084.957753735187, "last_run_timestamp": "2026-10-24T16:01:29.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8192149207920791, "accuracy_std_dev": 0.018823182524040232, "run_count": 6, "total_questions": 909, "avg_cost_per_task_usd": 0.11224348184818483, "avg_duration_per_task_ms": 401079.6094609461, "avg_output_tokens_per_task": 46532.80198019802, "last_run_timestamp": "2026-10-17T16:06:46.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8191437142857143, "accuracy_std_dev": 0.03986796895724072, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04288789971139971, "avg_duration_per_task_ms": 992008.3816738817, "avg_output_tokens_per_task": 26956.146464646463, "last_run_timestamp": "2026-10-17T08:20:47.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8191170000000002, "accuracy_std_dev": 0.01945730630551585, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006372274410774412, "avg_duration_per_task_ms": 464617.35016835015, "avg_output_tokens_per_task": 18796.420314253646, "last_run_timestamp": "2026-10-24T16:31:34.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.8184965756676558, "accuracy_std_dev": 0.008315500000000031, "run_count": 2, "total_questions": 337, "avg_cost_per_task_usd": 0.010153243323442137, "avg_duration_per_task_ms": 1501083.5370919881, "avg_output_tokens_per_task": 28895.703264094955, "last_run_timestamp": "2026-10-24T08:27:40.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8183501, "accuracy_std_dev": 0.014134440253862193, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.033925831313131316, "avg_duration_per_task_ms": 215707.2409090909, "avg_output_tokens_per_task": 13837.719191919192, "last_run_timestamp": "2026-10-24T08:16:59.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8181818, "accuracy_std_dev": 0.02137456277353997, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1023312393939394, "avg_duration_per_task_ms": 562383.997979798, "avg_output_tokens_per_task": 37995.12222222222, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.818143387341772, "accuracy_std_dev": 0.008051034079545293, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.03837104746835443, "avg_duration_per_task_ms": 216584.66329113924, "avg_output_tokens_per_task": 14732.209493670885, "last_run_timestamp": "2026-10-24T08:26:33.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8177376144306652, "accuracy_std_dev": 0.03151916867911054, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.013423886696730551, "avg_duration_per_task_ms": 855848.4718151072, "avg_output_tokens_per_task": 39679.556933483655, "last_run_timestamp": "2026-10-17T08:19:03.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8174118475609755, "accuracy_std_dev": 0.019551780804826967, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.02269767581300813, "avg_duration_per_task_ms": 492334.5833333333, "avg_output_tokens_per_task": 22647.941056910568, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.8173077377622378, "accuracy_std_dev": 0.04183386114142466, "run_count": 3, "total_questions": 572, "avg_cost_per_task_usd": 0.00918055944055944, "avg_duration_per_task_ms": 556054.2482517483, "avg_output_tokens_per_task": 20397.216783216783, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8172464444444444, "accuracy_std_dev": 0.007234812308263586, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.027580057800224465, "avg_duration_per_task_ms": 186978.31425364758, "avg_output_tokens_per_task": 23463.768237934906, "last_run_timestamp": "2026-10-24T16:20:59.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8170044527027026, "accuracy_std_dev": 0.0041701025034031, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.010224835585585587, "avg_duration_per_task_ms": 768325.4386261262, "avg_output_tokens_per_task": 18282.219594594593, "last_run_timestamp": "2026-10-24T16:12:04.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8167087499999999, "accuracy_std_dev": 0.01709464425448801, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007263619318181818, "avg_duration_per_task_ms": 512210.79671717173, "avg_output_tokens_per_task": 18196.01452020202, "last_run_timestamp": "2026-10-24T16:06:30.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8166852222222222, "accuracy_std_dev": 0.009167637558011412, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03448444051627385, "avg_duration_per_task_ms": 613993.6262626263, "avg_output_tokens_per_task": 16126.99775533109, "last_run_timestamp": "2026-10-24T09:01:52.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8166289763513515, "accuracy_std_dev": 0.02933294963006619, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.08447106756756757, "avg_duration_per_task_ms": 968983.0568693693, "avg_output_tokens_per_task": 41870.689752252256, "last_run_timestamp": "2026-10-24T16:15:14.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8162661034482758, "accuracy_std_dev": 0.01119088698017683, "run_count": 29, "total_questions": 5742, "avg_cost_per_task_usd": 0.33315613949843254, "avg_duration_per_task_ms": 297815.49094392196, "avg_output_tokens_per_task": 21074.363636363636, "last_run_timestamp": "2026-10-24T08:56:14.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8160686215384615, "accuracy_std_dev": 0.03359599240742859, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.026009656410256408, "avg_duration_per_task_ms": 463494.8851282051, "avg_output_tokens_per_task": 21626.916923076922, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.815937, "accuracy_std_dev": 0.021042110493009022, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04137441245791246, "avg_duration_per_task_ms": 82874.20875420875, "avg_output_tokens_per_task": 27465.196969696968, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.815488, "accuracy_std_dev": 0.010410666606898895, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.01878690909090909, "avg_duration_per_task_ms": 548800.3747474748, "avg_output_tokens_per_task": 18588.853535353537, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.815286480254777, "accuracy_std_dev": 0.006665997576319691, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.08723442165605096, "avg_duration_per_task_ms": 185900.4152866242, "avg_output_tokens_per_task": 24915.47388535032, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8152233565365026, "accuracy_std_dev": 0.024345312062738052, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.21931891850594232, "avg_duration_per_task_ms": 991109.1279003961, "avg_output_tokens_per_task": 59192.10016977929, "last_run_timestamp": "2026-10-24T16:04:12.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8151274695945945, "accuracy_std_dev": 0.04695932867392988, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.2281274650900901, "avg_duration_per_task_ms": 1121263.125563063, "avg_output_tokens_per_task": 52145.57432432433, "last_run_timestamp": "2026-10-24T16:06:30.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.814815, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006498378787878787, "avg_duration_per_task_ms": 589595.8333333334, "avg_output_tokens_per_task": 15681.459595959595, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.814815, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.007393566137566137, "avg_duration_per_task_ms": 549965.8571428572, "avg_output_tokens_per_task": 17741.814814814814, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.8147209999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.024774192893401017, "avg_duration_per_task_ms": 272533.6497461929, "avg_output_tokens_per_task": 20619.17766497462, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8142967307692308, "accuracy_std_dev": 0.010500659783511362, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.36391435470085476, "avg_duration_per_task_ms": 358484.0462315462, "avg_output_tokens_per_task": 22920.8277000777, "last_run_timestamp": "2026-10-24T08:44:41.412Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.8138046, "accuracy_std_dev": 0.01492933679169976, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010332651515151515, "avg_duration_per_task_ms": 579885.2626262626, "avg_output_tokens_per_task": 12515.30505050505, "last_run_timestamp": "2026-10-24T08:57:37.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8131313333333333, "accuracy_std_dev": 0.02212166805545088, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2550713585858586, "avg_duration_per_task_ms": 227479.37878787878, "avg_output_tokens_per_task": 25491.23232323232, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.8130656097285068, "accuracy_std_dev": 0.09667022957112152, "run_count": 9, "total_questions": 1768, "avg_cost_per_task_usd": 0.061109888574660626, "avg_duration_per_task_ms": 1704990.4128959277, "avg_output_tokens_per_task": 19426.679864253394, "last_run_timestamp": "2026-10-24T16:10:43.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.813051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.00766168253968254, "avg_duration_per_task_ms": 356938.3703703704, "avg_output_tokens_per_task": 33904.37566137566, "last_run_timestamp": "2026-10-14T18:00:03.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.8127462577873253, "accuracy_std_dev": 0.009366182671718485, "run_count": 5, "total_questions": 931, "avg_cost_per_task_usd": 0.043591290010741135, "avg_duration_per_task_ms": 179665.51987110634, "avg_output_tokens_per_task": 17266.0977443609, "last_run_timestamp": "2026-10-24T08:11:11.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8124578000000001, "accuracy_std_dev": 0.006784287240381237, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00604990505050505, "avg_duration_per_task_ms": 545364.5707070708, "avg_output_tokens_per_task": 18416.73232323232, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8122996179966044, "accuracy_std_dev": 0.02424420574075381, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.26123569835880023, "avg_duration_per_task_ms": 965363.1103565365, "avg_output_tokens_per_task": 59428.235427277876, "last_run_timestamp": "2026-10-24T16:00:53.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.8122604367816092, "accuracy_std_dev": 0.03327589059946405, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.024626680715197958, "avg_duration_per_task_ms": 153695.50446998724, "avg_output_tokens_per_task": 14302.206896551725, "last_run_timestamp": "2026-10-24T08:18:45.292Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.812079, "accuracy_std_dev": 0.1279844797709863, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15244413194444442, "avg_duration_per_task_ms": 1980931.956439394, "avg_output_tokens_per_task": 55523.080808080806, "last_run_timestamp": "2026-10-24T16:14:24.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8118685, "accuracy_std_dev": 0.0618254994258033, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.042041156565656566, "avg_duration_per_task_ms": 1127766.4457070706, "avg_output_tokens_per_task": 24710.378787878788, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8114476666666668, "accuracy_std_dev": 0.024344661016329636, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00885003591470258, "avg_duration_per_task_ms": 567443.1509539842, "avg_output_tokens_per_task": 18949.043209876545, "last_run_timestamp": "2026-10-24T16:00:55.675Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8113425625, "accuracy_std_dev": 0.01663880630096684, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.34052509564393935, "avg_duration_per_task_ms": 354477.9971590909, "avg_output_tokens_per_task": 22529.18023989899, "last_run_timestamp": "2026-10-24T08:55:49.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8111930073800738, "accuracy_std_dev": 0.03224095604623893, "run_count": 7, "total_questions": 813, "avg_cost_per_task_usd": 0.05198734809348094, "avg_duration_per_task_ms": 309460.76998769987, "avg_output_tokens_per_task": 20725.47355473555, "last_run_timestamp": "2026-10-24T16:02:08.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8103253333333333, "accuracy_std_dev": 0.011527714209195522, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04107354882154882, "avg_duration_per_task_ms": 75169.15151515152, "avg_output_tokens_per_task": 27258.996632996634, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8099945799011532, "accuracy_std_dev": 0.10451583165401959, "run_count": 13, "total_questions": 2428, "avg_cost_per_task_usd": 0.042656735584843494, "avg_duration_per_task_ms": 161924.82866556838, "avg_output_tokens_per_task": 20413.041598023065, "last_run_timestamp": "2026-10-24T16:12:07.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8091166153846153, "accuracy_std_dev": 0.03183613126622014, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.19830280854700855, "avg_duration_per_task_ms": 1060541.5008547008, "avg_output_tokens_per_task": 55592.3264957265, "last_run_timestamp": "2026-10-03T16:58:49.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.8089225, "accuracy_std_dev": 0.07491550000000002, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023557068181818182, "avg_duration_per_task_ms": 701295.6338383838, "avg_output_tokens_per_task": 58706.843434343435, "last_run_timestamp": "2026-10-24T08:01:41.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8089224, "accuracy_std_dev": 0.023427325085036924, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.11485773383838385, "avg_duration_per_task_ms": 769661.1737373738, "avg_output_tokens_per_task": 59732.906565656565, "last_run_timestamp": "2026-10-24T16:03:25.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8082103076923077, "accuracy_std_dev": 0.02014208867935159, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.3422723123543123, "avg_duration_per_task_ms": 356747.24864024867, "avg_output_tokens_per_task": 22654.46153846154, "last_run_timestamp": "2026-10-24T08:14:36.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.808081, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016894853535353537, "avg_duration_per_task_ms": 156500.61111111112, "avg_output_tokens_per_task": 6566.772727272727, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.808081, "accuracy_std_dev": 0.018027342695657245, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.044340436026936034, "avg_duration_per_task_ms": 217888.1734006734, "avg_output_tokens_per_task": 22075.030303030304, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.8069390466155811, "accuracy_std_dev": 0.09137699015060836, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.0966928748403576, "avg_duration_per_task_ms": 758869.1143039592, "avg_output_tokens_per_task": 28380.814814814814, "last_run_timestamp": "2026-10-24T16:10:43.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.806699224509804, "accuracy_std_dev": 0.027467920997381046, "run_count": 7, "total_questions": 1020, "avg_cost_per_task_usd": 0.07044824803921569, "avg_duration_per_task_ms": 1518236.9009803922, "avg_output_tokens_per_task": 40306.42941176471, "last_run_timestamp": "2026-10-24T16:26:52.183Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.8066778333333334, "accuracy_std_dev": 0.00898293258382558, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04676386700336701, "avg_duration_per_task_ms": 57672.9595959596, "avg_output_tokens_per_task": 15383.77946127946, "last_run_timestamp": "2026-10-24T08:45:03.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8065842222222221, "accuracy_std_dev": 0.02588009608765339, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.039878716610549945, "avg_duration_per_task_ms": 183894.81369248035, "avg_output_tokens_per_task": 22053.50448933782, "last_run_timestamp": "2026-10-24T16:05:36.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.806210222222222, "accuracy_std_dev": 0.03697273211969953, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.25474633557800225, "avg_duration_per_task_ms": 231596.29349046017, "avg_output_tokens_per_task": 25466.63243546577, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.8058380621468926, "accuracy_std_dev": 0.060531258531109376, "run_count": 5, "total_questions": 885, "avg_cost_per_task_usd": 0.012335033898305085, "avg_duration_per_task_ms": 403352.9514124294, "avg_output_tokens_per_task": 16083.899435028248, "last_run_timestamp": "2026-10-17T16:05:00.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.8057394503311259, "accuracy_std_dev": 0.027315871855937524, "run_count": 4, "total_questions": 604, "avg_cost_per_task_usd": 0.04681624668874173, "avg_duration_per_task_ms": 685385.5794701987, "avg_output_tokens_per_task": 20286.42715231788, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.805345125, "accuracy_std_dev": 0.036272944774437245, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19822186489898988, "avg_duration_per_task_ms": 832606.8560606061, "avg_output_tokens_per_task": 49537.894570707074, "last_run_timestamp": "2026-10-24T16:31:00.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.8049668265895952, "accuracy_std_dev": 0.04314684443397195, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.14483265382145152, "avg_duration_per_task_ms": 1590993.8619139371, "avg_output_tokens_per_task": 47443.71419396275, "last_run_timestamp": "2026-10-24T16:00:53.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8048667272727271, "accuracy_std_dev": 0.020423284626269588, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.33759371303948577, "avg_duration_per_task_ms": 354437.65059687785, "avg_output_tokens_per_task": 22371.59963269054, "last_run_timestamp": "2026-10-24T08:45:54.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8045734999999999, "accuracy_std_dev": 0.09801590415888298, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09265821506734007, "avg_duration_per_task_ms": 514476.9250841751, "avg_output_tokens_per_task": 34180.35227272727, "last_run_timestamp": "2026-10-24T16:04:33.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.8043401184631804, "accuracy_std_dev": 0.024791902944308233, "run_count": 5, "total_questions": 937, "avg_cost_per_task_usd": 0.03799079615795091, "avg_duration_per_task_ms": 223986.25720384205, "avg_output_tokens_per_task": 37938.2166488794, "last_run_timestamp": "2026-10-24T16:04:55.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.802714625, "accuracy_std_dev": 0.02322035202111231, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.028997014520202022, "avg_duration_per_task_ms": 309649.94507575757, "avg_output_tokens_per_task": 19185.933712121212, "last_run_timestamp": "2026-10-17T16:09:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8020951111111111, "accuracy_std_dev": 0.027492606765554845, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04329651571268237, "avg_duration_per_task_ms": 84651.98765432098, "avg_output_tokens_per_task": 26957.542087542086, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.8011755539419088, "accuracy_std_dev": 0.08103010676902754, "run_count": 8, "total_questions": 1446, "avg_cost_per_task_usd": 0.018679138312586448, "avg_duration_per_task_ms": 300257.92392807745, "avg_output_tokens_per_task": 39516.01521438451, "last_run_timestamp": "2026-10-24T08:32:39.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.8007853333333332, "accuracy_std_dev": 0.008398773614178595, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0439676531986532, "avg_duration_per_task_ms": 201662.1632996633, "avg_output_tokens_per_task": 21888.12962962963, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.8005984444444443, "accuracy_std_dev": 0.04083905003278416, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.026883831649831647, "avg_duration_per_task_ms": 331939.1515151515, "avg_output_tokens_per_task": 25611.442199775534, "last_run_timestamp": "2026-10-24T16:03:25.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7996631666666666, "accuracy_std_dev": 0.024569953550469384, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.045980551346801346, "avg_duration_per_task_ms": 194150.27525252526, "avg_output_tokens_per_task": 21808.128787878788, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7995168260869566, "accuracy_std_dev": 0.015725089308490422, "run_count": 5, "total_questions": 966, "avg_cost_per_task_usd": 0.1727357877846791, "avg_duration_per_task_ms": 863549.4824016563, "avg_output_tokens_per_task": 55062.07867494824, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.798727888888889, "accuracy_std_dev": 0.10517670417967454, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05109221941638608, "avg_duration_per_task_ms": 1276720.367003367, "avg_output_tokens_per_task": 27966.93265993266, "last_run_timestamp": "2026-10-24T16:17:39.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.798541, "accuracy_std_dev": 0.005382643216859184, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013865464646464647, "avg_duration_per_task_ms": 310914.7727272727, "avg_output_tokens_per_task": 14252.202861952863, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7979798333333333, "accuracy_std_dev": 0.04819829830819293, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1916180345117845, "avg_duration_per_task_ms": 1503115.5151515151, "avg_output_tokens_per_task": 46476.21548821549, "last_run_timestamp": "2026-10-03T17:00:45.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7975339581632652, "accuracy_std_dev": 0.14314198310659945, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.01657647346938776, "avg_duration_per_task_ms": 547902.756122449, "avg_output_tokens_per_task": 26479.933673469386, "last_run_timestamp": "2026-10-17T08:10:43.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7969696000000001, "accuracy_std_dev": 0.010464901654578508, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.013516212121212122, "avg_duration_per_task_ms": 270529.88181818184, "avg_output_tokens_per_task": 14086.99090909091, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.796581, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.06800715384615384, "avg_duration_per_task_ms": 384890.6564102564, "avg_output_tokens_per_task": 29421.871794871793, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7964659214659687, "accuracy_std_dev": 0.005460143336030668, "run_count": 8, "total_questions": 1528, "avg_cost_per_task_usd": 0.050215497382198944, "avg_duration_per_task_ms": 1089140.4594240838, "avg_output_tokens_per_task": 22865.37238219895, "last_run_timestamp": "2026-10-17T16:06:10.469Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7956228, "accuracy_std_dev": 0.021522407276138952, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006844225252525253, "avg_duration_per_task_ms": 363476.91515151516, "avg_output_tokens_per_task": 16652.98888888889, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.7950335, "accuracy_std_dev": 0.010057183614213304, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.053936401515151514, "avg_duration_per_task_ms": 418610.5353535354, "avg_output_tokens_per_task": 23400.56818181818, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.7950015744680852, "accuracy_std_dev": 0.020697025299303277, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.028578175278622088, "avg_duration_per_task_ms": 466139.9655521783, "avg_output_tokens_per_task": 23874.156028368794, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7944022499999999, "accuracy_std_dev": 0.03928997983185408, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.04325068055555555, "avg_duration_per_task_ms": 495166.12626262626, "avg_output_tokens_per_task": 42549.15993265993, "last_run_timestamp": "2026-10-24T16:02:47.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7942762, "accuracy_std_dev": 0.008274731256059032, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014654871717171718, "avg_duration_per_task_ms": 281996.1565656566, "avg_output_tokens_per_task": 14013.025252525253, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7940515555555555, "accuracy_std_dev": 0.013561145494972105, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.017693217171717172, "avg_duration_per_task_ms": 59941.32828282828, "avg_output_tokens_per_task": 5742.835016835017, "last_run_timestamp": "2026-10-24T16:02:28.570Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7929315464581417, "accuracy_std_dev": 0.02021084770060026, "run_count": 11, "total_questions": 2174, "avg_cost_per_task_usd": 0.06395397148114075, "avg_duration_per_task_ms": 189643.35372585096, "avg_output_tokens_per_task": 20031.46780128795, "last_run_timestamp": "2026-10-24T08:53:01.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7927813197452228, "accuracy_std_dev": 0.008711832686065535, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.011660411464968153, "avg_duration_per_task_ms": 735190.7324840764, "avg_output_tokens_per_task": 39161.06751592357, "last_run_timestamp": "2026-10-24T16:38:44.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.79250825, "accuracy_std_dev": 0.011190804686028344, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.004598987794612795, "avg_duration_per_task_ms": 248268.39141414143, "avg_output_tokens_per_task": 30111.32239057239, "last_run_timestamp": "2026-10-17T16:03:34.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7920900579268293, "accuracy_std_dev": 0.011512971285033247, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.01824498475609756, "avg_duration_per_task_ms": 181543.2469512195, "avg_output_tokens_per_task": 15051.83943089431, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.79194725, "accuracy_std_dev": 0.03294298282417921, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.10781298148148147, "avg_duration_per_task_ms": 400979.42887205386, "avg_output_tokens_per_task": 29714.260101010103, "last_run_timestamp": "2026-10-24T16:36:23.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7919261222410866, "accuracy_std_dev": 0.024709971729827782, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.29162476117713637, "avg_duration_per_task_ms": 950106.2246745897, "avg_output_tokens_per_task": 73483.63723825694, "last_run_timestamp": "2026-10-24T16:02:08.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7914427791218638, "accuracy_std_dev": 0.014961655425666921, "run_count": 12, "total_questions": 2232, "avg_cost_per_task_usd": 0.004660359318996416, "avg_duration_per_task_ms": 1335489.0049283155, "avg_output_tokens_per_task": 30608.047491039426, "last_run_timestamp": "2026-10-24T16:07:10.479Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7902960850403475, "accuracy_std_dev": 0.020316089842022662, "run_count": 9, "total_questions": 1611, "avg_cost_per_task_usd": 0.040685735567970206, "avg_duration_per_task_ms": 737944.181874612, "avg_output_tokens_per_task": 18801.04593420236, "last_run_timestamp": "2026-10-17T16:06:15.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7893355839160839, "accuracy_std_dev": 0.10239971274711347, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.07681546794871794, "avg_duration_per_task_ms": 733179.0168997669, "avg_output_tokens_per_task": 27895.818764568765, "last_run_timestamp": "2026-10-10T16:29:40.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7891638401060072, "accuracy_std_dev": 0.014821929173009913, "run_count": 12, "total_questions": 2264, "avg_cost_per_task_usd": 0.008643914752650177, "avg_duration_per_task_ms": 377329.0070671378, "avg_output_tokens_per_task": 34148.09540636042, "last_run_timestamp": "2026-10-24T16:13:05.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7887112722044729, "accuracy_std_dev": 0.11519716273394331, "run_count": 9, "total_questions": 1565, "avg_cost_per_task_usd": 0.03999107923322684, "avg_duration_per_task_ms": 140844.86581469647, "avg_output_tokens_per_task": 29636.458785942494, "last_run_timestamp": "2026-10-24T16:27:33.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7885339041353384, "accuracy_std_dev": 0.023230108247703007, "run_count": 5, "total_questions": 532, "avg_cost_per_task_usd": 0.034030992481203004, "avg_duration_per_task_ms": 116832.8590225564, "avg_output_tokens_per_task": 35347.6954887218, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7878787777777778, "accuracy_std_dev": 0.04255992961506759, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17480242087542086, "avg_duration_per_task_ms": 541272.2081930415, "avg_output_tokens_per_task": 54419.242424242424, "last_run_timestamp": "2026-10-24T16:04:55.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7854672791234141, "accuracy_std_dev": 0.12208478471357521, "run_count": 5, "total_questions": 867, "avg_cost_per_task_usd": 0.023352209919261825, "avg_duration_per_task_ms": 590708.8500576701, "avg_output_tokens_per_task": 19635.643598615916, "last_run_timestamp": "2026-10-17T08:18:07.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7851099187817259, "accuracy_std_dev": 0.024182763553029427, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.05363871517202482, "avg_duration_per_task_ms": 470093.5600676819, "avg_output_tokens_per_task": 23310.749576988157, "last_run_timestamp": "2026-10-17T16:03:34.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7849183946015424, "accuracy_std_dev": 0.0724374841967196, "run_count": 8, "total_questions": 1556, "avg_cost_per_task_usd": 0.1867961233933162, "avg_duration_per_task_ms": 1640932.23907455, "avg_output_tokens_per_task": 43985.5205655527, "last_run_timestamp": "2026-10-24T16:01:08.930Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7842683649508387, "accuracy_std_dev": 0.12512206342943477, "run_count": 9, "total_questions": 1729, "avg_cost_per_task_usd": 0.024085618276460385, "avg_duration_per_task_ms": 354469.05031810293, "avg_output_tokens_per_task": 24486.457489878543, "last_run_timestamp": "2026-10-24T16:05:36.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.7839820659898478, "accuracy_std_dev": 0.002844524682028023, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.013566983079526225, "avg_duration_per_task_ms": 498549.5752961083, "avg_output_tokens_per_task": 14135.162436548224, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7838284348819498, "accuracy_std_dev": 0.08896072103931436, "run_count": 11, "total_questions": 1313, "avg_cost_per_task_usd": 0.006210692307692306, "avg_duration_per_task_ms": 295622.3008377761, "avg_output_tokens_per_task": 24966.161462300075, "last_run_timestamp": "2026-10-24T16:40:33.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7836699999999999, "accuracy_std_dev": 0.010838720902394346, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.021049996969696963, "avg_duration_per_task_ms": 263607.26313131314, "avg_output_tokens_per_task": 12518.16212121212, "last_run_timestamp": "2026-10-24T16:18:15.010Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.7819865, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.028098762626262628, "avg_duration_per_task_ms": 185634.4823232323, "avg_output_tokens_per_task": 55958.419191919194, "last_run_timestamp": "2026-10-24T08:34:00.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7816675629587804, "accuracy_std_dev": 0.02991313237433426, "run_count": 9, "total_questions": 1771, "avg_cost_per_task_usd": 0.011191632411067194, "avg_duration_per_task_ms": 582007.9429700734, "avg_output_tokens_per_task": 37062.36589497459, "last_run_timestamp": "2026-10-24T16:18:35.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7808153280721533, "accuracy_std_dev": 0.05722357962705003, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.007362833145434047, "avg_duration_per_task_ms": 871339.3737316799, "avg_output_tokens_per_task": 18677.131341600903, "last_run_timestamp": "2026-10-24T16:21:54.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.7805836666666667, "accuracy_std_dev": 0.009152126540986119, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03482366666666667, "avg_duration_per_task_ms": 531300.7171717172, "avg_output_tokens_per_task": 69410.62962962964, "last_run_timestamp": "2026-10-24T08:41:52.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7805031924528302, "accuracy_std_dev": 0.020942354059550135, "run_count": 3, "total_questions": 265, "avg_cost_per_task_usd": 0.09049491698113207, "avg_duration_per_task_ms": 131957.70943396227, "avg_output_tokens_per_task": 37606.645283018865, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.780302875, "accuracy_std_dev": 0.044334001808537145, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.012861900883838383, "avg_duration_per_task_ms": 776646.4172979798, "avg_output_tokens_per_task": 38634.71654040404, "last_run_timestamp": "2026-10-17T08:18:06.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7786195454545455, "accuracy_std_dev": 0.04555204390657037, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.21966554178145087, "avg_duration_per_task_ms": 867448.9150596878, "avg_output_tokens_per_task": 73058.02662993572, "last_run_timestamp": "2026-10-17T16:06:46.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.7785086368421053, "accuracy_std_dev": 0.010062970594585871, "run_count": 4, "total_questions": 760, "avg_cost_per_task_usd": 0.002249036842105263, "avg_duration_per_task_ms": 780019.0210526315, "avg_output_tokens_per_task": 14812.598684210527, "last_run_timestamp": "2026-10-24T16:06:32.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7784476091370558, "accuracy_std_dev": 0.018688411382040342, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.022516517131979694, "avg_duration_per_task_ms": 871719.5628172589, "avg_output_tokens_per_task": 38655.122461928935, "last_run_timestamp": "2026-10-24T16:37:21.981Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7780583333333333, "accuracy_std_dev": 0.018220714966457528, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.021326852693602693, "avg_duration_per_task_ms": 472582.55723905726, "avg_output_tokens_per_task": 28353.299663299662, "last_run_timestamp": "2026-10-24T16:05:13.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7779681626712327, "accuracy_std_dev": 0.05316331589603613, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.15604383333333335, "avg_duration_per_task_ms": 982840.4474885844, "avg_output_tokens_per_task": 64456.35901826484, "last_run_timestamp": "2026-10-24T16:03:59.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7774583256704981, "accuracy_std_dev": 0.04658687436660674, "run_count": 4, "total_questions": 522, "avg_cost_per_task_usd": 0.1387438639846743, "avg_duration_per_task_ms": 1171411.5785440614, "avg_output_tokens_per_task": 40017.36590038314, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7772166666666667, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013080006734006736, "avg_duration_per_task_ms": 67697.39225589225, "avg_output_tokens_per_task": 10341.513468013469, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7769359999999998, "accuracy_std_dev": 0.005389920933248147, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00513745202020202, "avg_duration_per_task_ms": 836559.9427609427, "avg_output_tokens_per_task": 28379.09006734007, "last_run_timestamp": "2026-10-24T16:31:50.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7768158778004073, "accuracy_std_dev": 0.02614019988234382, "run_count": 6, "total_questions": 982, "avg_cost_per_task_usd": 0.010268270875763747, "avg_duration_per_task_ms": 1330121.382892057, "avg_output_tokens_per_task": 39680.64867617108, "last_run_timestamp": "2026-10-24T08:12:02.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7767955833333333, "accuracy_std_dev": 0.0196863031719109, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.005944730218855219, "avg_duration_per_task_ms": 663975.4595959596, "avg_output_tokens_per_task": 36414.121632996634, "last_run_timestamp": "2026-10-24T16:20:08.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.77567325, "accuracy_std_dev": 0.010934638822910419, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013401992424242422, "avg_duration_per_task_ms": 67621.48106060606, "avg_output_tokens_per_task": 10598.125, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7752524999999999, "accuracy_std_dev": 0.021901537394667075, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05750532828282828, "avg_duration_per_task_ms": 632332.7689393939, "avg_output_tokens_per_task": 23028.97095959596, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7747842799160252, "accuracy_std_dev": 0.014649286504468402, "run_count": 10, "total_questions": 1429, "avg_cost_per_task_usd": 0.04802459482155353, "avg_duration_per_task_ms": 974559.5045486355, "avg_output_tokens_per_task": 24459.735479356194, "last_run_timestamp": "2026-10-24T16:01:45.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7741301666666666, "accuracy_std_dev": 0.007235617536802828, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.019828150673400675, "avg_duration_per_task_ms": 721854.4562289562, "avg_output_tokens_per_task": 49359.81902356903, "last_run_timestamp": "2026-10-10T16:25:17.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7741139703903096, "accuracy_std_dev": 0.09718478875928833, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.004475604306864065, "avg_duration_per_task_ms": 312870.95827725437, "avg_output_tokens_per_task": 22926.07267833109, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7718855, "accuracy_std_dev": 0.03473679068005565, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05189419633838384, "avg_duration_per_task_ms": 871715.6748737374, "avg_output_tokens_per_task": 23212.847222222223, "last_run_timestamp": "2026-10-17T16:03:24.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7715289560632689, "accuracy_std_dev": 0.034441084737222544, "run_count": 9, "total_questions": 1138, "avg_cost_per_task_usd": 0.012996073813708259, "avg_duration_per_task_ms": 52972.56414762742, "avg_output_tokens_per_task": 17226.330404217926, "last_run_timestamp": "2026-10-24T16:14:06.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7714323076923078, "accuracy_std_dev": 0.009526863648769632, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.014571612276612276, "avg_duration_per_task_ms": 136246.84032634032, "avg_output_tokens_per_task": 29062.332167832166, "last_run_timestamp": "2026-10-24T16:26:32.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7707066962474647, "accuracy_std_dev": 0.0069177655641109856, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.011308173427991888, "avg_duration_per_task_ms": 972932.3559837728, "avg_output_tokens_per_task": 11764.48985801217, "last_run_timestamp": "2026-10-24T16:05:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.770202, "accuracy_std_dev": 0.008625232557251217, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013833271043771043, "avg_duration_per_task_ms": 119359.86616161616, "avg_output_tokens_per_task": 10422.487373737375, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.769679306122449, "accuracy_std_dev": 0.017686462624278497, "run_count": 5, "total_questions": 686, "avg_cost_per_task_usd": 0.009246491253644315, "avg_duration_per_task_ms": 700873.1428571428, "avg_output_tokens_per_task": 30693.99271137026, "last_run_timestamp": "2026-10-24T16:14:24.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7691307272727272, "accuracy_std_dev": 0.12910889092269848, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.016461994031221304, "avg_duration_per_task_ms": 677663.191919192, "avg_output_tokens_per_task": 35541.38842975206, "last_run_timestamp": "2026-10-24T08:57:59.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.768994582345191, "accuracy_std_dev": 0.016269118990725185, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.021888522397891963, "avg_duration_per_task_ms": 236418.71277997366, "avg_output_tokens_per_task": 29098.49209486166, "last_run_timestamp": "2026-10-24T16:34:51.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7689713726812815, "accuracy_std_dev": 0.07621573205272868, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.018058317875210794, "avg_duration_per_task_ms": 1377820.0623946036, "avg_output_tokens_per_task": 17446.093591905566, "last_run_timestamp": "2026-10-24T16:01:29.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7685185, "accuracy_std_dev": 0.018461144134370444, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01486243308080808, "avg_duration_per_task_ms": 1091396.6527777778, "avg_output_tokens_per_task": 52782.86363636364, "last_run_timestamp": "2026-10-24T08:54:50.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7677873143148243, "accuracy_std_dev": 0.10954529486385985, "run_count": 10, "total_questions": 1963, "avg_cost_per_task_usd": 0.15105290779419256, "avg_duration_per_task_ms": 869942.4396332145, "avg_output_tokens_per_task": 39571.91288843607, "last_run_timestamp": "2026-10-24T16:19:11.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7676768, "accuracy_std_dev": 0.018594973916625974, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015932259595959595, "avg_duration_per_task_ms": 441129.9383838384, "avg_output_tokens_per_task": 16024.338383838383, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7676733942992873, "accuracy_std_dev": 0.016408032276093717, "run_count": 15, "total_questions": 2947, "avg_cost_per_task_usd": 0.00717780895826264, "avg_duration_per_task_ms": 441831.9945707499, "avg_output_tokens_per_task": 27893.447234475738, "last_run_timestamp": "2026-10-24T16:03:06.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7671445564924115, "accuracy_std_dev": 0.01261567820104104, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.019929450252951095, "avg_duration_per_task_ms": 1212404.510961214, "avg_output_tokens_per_task": 60253.559021922425, "last_run_timestamp": "2026-10-24T16:08:41.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7666666510144927, "accuracy_std_dev": 0.008075638536845108, "run_count": 9, "total_questions": 1725, "avg_cost_per_task_usd": 0.02488580347826087, "avg_duration_per_task_ms": 72284.5831884058, "avg_output_tokens_per_task": 26060.85043478261, "last_run_timestamp": "2026-10-24T16:10:43.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.76599325, "accuracy_std_dev": 0.026856781949584005, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.012383484848484849, "avg_duration_per_task_ms": 910556.1414141414, "avg_output_tokens_per_task": 45554.207070707074, "last_run_timestamp": "2026-10-24T08:15:51.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7659344612403101, "accuracy_std_dev": 0.01074604027072297, "run_count": 4, "total_questions": 774, "avg_cost_per_task_usd": 0.004193177002583979, "avg_duration_per_task_ms": 4103.266149870801, "avg_output_tokens_per_task": 2323.702842377261, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7651274895635675, "accuracy_std_dev": 0.012168813200753945, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.01713221790006325, "avg_duration_per_task_ms": 104832.72896900696, "avg_output_tokens_per_task": 5987.342504743833, "last_run_timestamp": "2026-10-24T09:05:10.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7646065075987841, "accuracy_std_dev": 0.016409967770839785, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.056628046605876385, "avg_duration_per_task_ms": 583978.3080040527, "avg_output_tokens_per_task": 26120.428571428572, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.76431, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017299227272727275, "avg_duration_per_task_ms": 73383.17171717172, "avg_output_tokens_per_task": 5944.843434343435, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7632574999999999, "accuracy_std_dev": 0.011447194448859535, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.020826034090909093, "avg_duration_per_task_ms": 1150567.0700757576, "avg_output_tokens_per_task": 51976.968434343435, "last_run_timestamp": "2026-10-24T16:20:24.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7631440720576461, "accuracy_std_dev": 0.033247584998763444, "run_count": 8, "total_questions": 1249, "avg_cost_per_task_usd": 0.06387943394715773, "avg_duration_per_task_ms": 833616.5316253003, "avg_output_tokens_per_task": 53539.40272217774, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.7628913204419889, "accuracy_std_dev": 0.046742499999999965, "run_count": 2, "total_questions": 362, "avg_cost_per_task_usd": 0.015507096685082874, "avg_duration_per_task_ms": 476532.59392265195, "avg_output_tokens_per_task": 53949.68508287293, "last_run_timestamp": "2026-10-24T08:33:50.635Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.7626265, "accuracy_std_dev": 0.007714766182458135, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0075155000000000005, "avg_duration_per_task_ms": 129316.82575757576, "avg_output_tokens_per_task": 29867.227272727272, "last_run_timestamp": "2026-10-24T16:12:49.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.762346, "accuracy_std_dev": 0.0014306672103136615, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004426390572390572, "avg_duration_per_task_ms": 502309.91414141416, "avg_output_tokens_per_task": 29382.90404040404, "last_run_timestamp": "2026-10-24T16:30:53.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7616160933333332, "accuracy_std_dev": 0.0330626227537687, "run_count": 9, "total_questions": 1650, "avg_cost_per_task_usd": 0.0073882921212121205, "avg_duration_per_task_ms": 943448.4084848484, "avg_output_tokens_per_task": 19044.426666666666, "last_run_timestamp": "2026-10-24T16:00:43.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7605754720812183, "accuracy_std_dev": 0.010447569992586773, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0025078527918781725, "avg_duration_per_task_ms": 100160.06345177664, "avg_output_tokens_per_task": 31388.34771573604, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7603815555555556, "accuracy_std_dev": 0.014085160481956509, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.01835896689113356, "avg_duration_per_task_ms": 156375.88019079686, "avg_output_tokens_per_task": 30374.710718294053, "last_run_timestamp": "2026-10-24T16:26:58.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7603039230769231, "accuracy_std_dev": 0.042288769102708845, "run_count": 9, "total_questions": 1755, "avg_cost_per_task_usd": 0.27848958746438746, "avg_duration_per_task_ms": 886078.2860398861, "avg_output_tokens_per_task": 74626.49743589743, "last_run_timestamp": "2026-10-24T16:00:51.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.7602694000000001, "accuracy_std_dev": 0.013238766658567559, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01848461414141414, "avg_duration_per_task_ms": 133542.18232323232, "avg_output_tokens_per_task": 30573.586868686867, "last_run_timestamp": "2026-10-24T16:05:36.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7596333333333335, "accuracy_std_dev": 0.006337831034527698, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.017320402918069584, "avg_duration_per_task_ms": 74089.44051627385, "avg_output_tokens_per_task": 28627.265432098764, "last_run_timestamp": "2026-10-24T16:15:45.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7592592222222222, "accuracy_std_dev": 0.13656078835349622, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02930702805836139, "avg_duration_per_task_ms": 133789.24130190795, "avg_output_tokens_per_task": 22446.539842873175, "last_run_timestamp": "2026-10-24T16:02:08.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.758698, "accuracy_std_dev": 0.012009662443216293, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.056970961279461274, "avg_duration_per_task_ms": 99388.35016835017, "avg_output_tokens_per_task": 22699.07239057239, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.757883010135135, "accuracy_std_dev": 0.030369548845212157, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.004753131193693694, "avg_duration_per_task_ms": 685197.0304054054, "avg_output_tokens_per_task": 28398.651463963964, "last_run_timestamp": "2026-10-24T16:26:55.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7575756666666668, "accuracy_std_dev": 0.0064470965471977405, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022524166666666668, "avg_duration_per_task_ms": 96254.67171717172, "avg_output_tokens_per_task": 14855.659932659933, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7569795193929174, "accuracy_std_dev": 0.12787048553193514, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.006095398538504777, "avg_duration_per_task_ms": 396590.173693086, "avg_output_tokens_per_task": 18629.006745362563, "last_run_timestamp": "2026-10-24T16:01:05.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7566855118644068, "accuracy_std_dev": 0.014912476720874281, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.016834027683615818, "avg_duration_per_task_ms": 248403.6107344633, "avg_output_tokens_per_task": 28301.17288135593, "last_run_timestamp": "2026-10-24T16:36:35.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7564534444444445, "accuracy_std_dev": 0.04085365377950671, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05544543434343434, "avg_duration_per_task_ms": 998107.7586980921, "avg_output_tokens_per_task": 28448.752525252527, "last_run_timestamp": "2026-10-24T16:02:06.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7557063684210527, "accuracy_std_dev": 0.055628184844211205, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.2860427289190719, "avg_duration_per_task_ms": 984270.246179966, "avg_output_tokens_per_task": 83930.79456706283, "last_run_timestamp": "2026-10-24T16:08:29.474Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7547698333333334, "accuracy_std_dev": 0.0766049855914889, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004288400673400673, "avg_duration_per_task_ms": 447675.2971380471, "avg_output_tokens_per_task": 15291.021885521886, "last_run_timestamp": "2026-10-17T08:26:03.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.754543774030354, "accuracy_std_dev": 0.030380293532339533, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.17601991231028669, "avg_duration_per_task_ms": 1058551.6509274873, "avg_output_tokens_per_task": 79756.01405283867, "last_run_timestamp": "2026-10-24T16:01:48.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7544054846967168, "accuracy_std_dev": 0.03260817390833722, "run_count": 11, "total_questions": 1797, "avg_cost_per_task_usd": 0.045709284919309964, "avg_duration_per_task_ms": 262995.0717863105, "avg_output_tokens_per_task": 17224.77796327212, "last_run_timestamp": "2026-10-24T08:41:25.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.753788, "accuracy_std_dev": 0.02000377094449942, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07918905176767678, "avg_duration_per_task_ms": 855570.0353535353, "avg_output_tokens_per_task": 22639.340909090908, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7516834000000001, "accuracy_std_dev": 0.0580270664952141, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01393037222222222, "avg_duration_per_task_ms": 97444.4191919192, "avg_output_tokens_per_task": 5974.968181818182, "last_run_timestamp": "2026-10-24T08:48:00.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.75084367721519, "accuracy_std_dev": 0.024667232906226022, "run_count": 4, "total_questions": 790, "avg_cost_per_task_usd": 0.020310108860759495, "avg_duration_per_task_ms": 155074.11012658227, "avg_output_tokens_per_task": 11752.967088607595, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7508415000000002, "accuracy_std_dev": 0.0622895, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06288509848484848, "avg_duration_per_task_ms": 493754.3712121212, "avg_output_tokens_per_task": 26228.825757575756, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.75076231097561, "accuracy_std_dev": 0.10261490128027215, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.006507009654471546, "avg_duration_per_task_ms": 566183.218495935, "avg_output_tokens_per_task": 27810.678353658535, "last_run_timestamp": "2026-10-10T08:22:53.947Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.7493987142857143, "accuracy_std_dev": 0.00923961159842752, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.013690483405483407, "avg_duration_per_task_ms": 96167.38383838384, "avg_output_tokens_per_task": 15049.160173160173, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7485971111111113, "accuracy_std_dev": 0.012497660327210088, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.017153025252525253, "avg_duration_per_task_ms": 1346127.6537598204, "avg_output_tokens_per_task": 55391.88832772166, "last_run_timestamp": "2026-10-17T16:05:03.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.7481013645569621, "accuracy_std_dev": 0.004003499999999993, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.010130237974683544, "avg_duration_per_task_ms": 298779.6734177215, "avg_output_tokens_per_task": 60721.07341772152, "last_run_timestamp": "2026-10-10T08:39:12.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7477555, "accuracy_std_dev": 0.009932117124930308, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01480825, "avg_duration_per_task_ms": 108491.74326599327, "avg_output_tokens_per_task": 14487.641414141413, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7477012353448277, "accuracy_std_dev": 0.024896992942589145, "run_count": 6, "total_questions": 1160, "avg_cost_per_task_usd": 0.016306832758620687, "avg_duration_per_task_ms": 316542.88879310346, "avg_output_tokens_per_task": 52791.42327586207, "last_run_timestamp": "2026-10-24T08:12:52.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018268570707070705, "avg_duration_per_task_ms": 31005.68686868687, "avg_output_tokens_per_task": 5873.606060606061, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7468014, "accuracy_std_dev": 0.009196272649285712, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015463989898989897, "avg_duration_per_task_ms": 342566.1424242424, "avg_output_tokens_per_task": 63071.326262626266, "last_run_timestamp": "2026-10-24T16:16:25.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7465043071718538, "accuracy_std_dev": 0.027768881506803414, "run_count": 6, "total_questions": 739, "avg_cost_per_task_usd": 0.053211952638700946, "avg_duration_per_task_ms": 424838.5182679296, "avg_output_tokens_per_task": 15554.012178619756, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7462993119730186, "accuracy_std_dev": 0.0800254867207247, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.00489628442945475, "avg_duration_per_task_ms": 596052.9263631253, "avg_output_tokens_per_task": 32164.360314783586, "last_run_timestamp": "2026-10-17T16:03:21.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7460317142857142, "accuracy_std_dev": 0.007918207277521612, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006722455266955267, "avg_duration_per_task_ms": 164422.91197691197, "avg_output_tokens_per_task": 7810.999278499278, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7459349573170732, "accuracy_std_dev": 0.02985061357091343, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.014618331300813008, "avg_duration_per_task_ms": 1032360.968495935, "avg_output_tokens_per_task": 51870.760162601626, "last_run_timestamp": "2026-10-24T08:20:32.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7441247969543147, "accuracy_std_dev": 0.013743685198646926, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.1143873355893965, "avg_duration_per_task_ms": 753346.6830231247, "avg_output_tokens_per_task": 44665.73547659334, "last_run_timestamp": "2026-10-24T16:39:40.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7435209222520109, "accuracy_std_dev": 0.004720778410977948, "run_count": 6, "total_questions": 1119, "avg_cost_per_task_usd": 0.008935632707774798, "avg_duration_per_task_ms": 139189.39052725647, "avg_output_tokens_per_task": 11357.61126005362, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.743266, "accuracy_std_dev": 0.006355106254029135, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005297381944444444, "avg_duration_per_task_ms": 155629.39141414143, "avg_output_tokens_per_task": 8558.156565656565, "last_run_timestamp": "2026-10-17T16:06:15.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.742214, "accuracy_std_dev": 0.015132597149861616, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01671199116161616, "avg_duration_per_task_ms": 766495.9797979798, "avg_output_tokens_per_task": 59349.530303030304, "last_run_timestamp": "2026-10-24T08:26:00.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7404603333333333, "accuracy_std_dev": 0.009642303833743399, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.023280140572390574, "avg_duration_per_task_ms": 99017.15824915825, "avg_output_tokens_per_task": 15411.767676767677, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7395181320432556, "accuracy_std_dev": 0.058919001127620475, "run_count": 9, "total_questions": 1757, "avg_cost_per_task_usd": 0.2870167250996016, "avg_duration_per_task_ms": 1318827.1798520205, "avg_output_tokens_per_task": 91069.35685828116, "last_run_timestamp": "2026-10-24T16:02:28.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.738309, "accuracy_std_dev": 0.008136215650479735, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.059655349046015715, "avg_duration_per_task_ms": 125601.6632996633, "avg_output_tokens_per_task": 23853.966891133558, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.737284008888889, "accuracy_std_dev": 0.05166831288604067, "run_count": 7, "total_questions": 1350, "avg_cost_per_task_usd": 0.017425666666666666, "avg_duration_per_task_ms": 990522.262962963, "avg_output_tokens_per_task": 44050.16814814815, "last_run_timestamp": "2026-10-24T16:13:05.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.7359708333333334, "accuracy_std_dev": 0.009394253179766615, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004828659932659932, "avg_duration_per_task_ms": 106903.15909090909, "avg_output_tokens_per_task": 7872.062289562289, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7349895701863354, "accuracy_std_dev": 0.017162011317797845, "run_count": 6, "total_questions": 805, "avg_cost_per_task_usd": 0.015730058385093167, "avg_duration_per_task_ms": 68792.66708074535, "avg_output_tokens_per_task": 9043.119254658384, "last_run_timestamp": "2026-10-17T16:03:44.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7349291052009457, "accuracy_std_dev": 0.007543459813017969, "run_count": 9, "total_questions": 1692, "avg_cost_per_task_usd": 0.006669232269503548, "avg_duration_per_task_ms": 243172.80910165486, "avg_output_tokens_per_task": 8248.872340425532, "last_run_timestamp": "2026-10-24T16:02:47.278Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7341193589990375, "accuracy_std_dev": 0.14473932746638518, "run_count": 11, "total_questions": 2078, "avg_cost_per_task_usd": 0.10010441241578441, "avg_duration_per_task_ms": 669272.0486044274, "avg_output_tokens_per_task": 29290.59720885467, "last_run_timestamp": "2026-10-24T16:11:49.038Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7340817802024036, "accuracy_std_dev": 0.01834012481629673, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.006654911448450348, "avg_duration_per_task_ms": 123137.50853889943, "avg_output_tokens_per_task": 6968.909550917141, "last_run_timestamp": "2026-10-17T16:03:44.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7333332203389831, "accuracy_std_dev": 0.014287186233444833, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.006560344632768362, "avg_duration_per_task_ms": 335200.51129943505, "avg_output_tokens_per_task": 26048.953107344634, "last_run_timestamp": "2026-10-24T16:16:05.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.7308918662420382, "accuracy_std_dev": 0.007761999999999991, "run_count": 2, "total_questions": 314, "avg_cost_per_task_usd": 0.007396050955414012, "avg_duration_per_task_ms": 197815.93949044586, "avg_output_tokens_per_task": 9168.713375796178, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7307692307692307, "accuracy_std_dev": 0.014845665656780083, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.18598931313131314, "avg_duration_per_task_ms": 298695.4405594406, "avg_output_tokens_per_task": 37027.50971250971, "last_run_timestamp": "2026-10-24T08:28:43.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7307283421052632, "accuracy_std_dev": 0.012478804160563647, "run_count": 38, "total_questions": 7524, "avg_cost_per_task_usd": 0.20430948484848477, "avg_duration_per_task_ms": 243835.08971291865, "avg_output_tokens_per_task": 38154.19364699628, "last_run_timestamp": "2026-10-24T08:29:16.808Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7299411591650359, "accuracy_std_dev": 0.0992559733114335, "run_count": 9, "total_questions": 1533, "avg_cost_per_task_usd": 0.00431450815394651, "avg_duration_per_task_ms": 554566.0965427267, "avg_output_tokens_per_task": 26096.888454011743, "last_run_timestamp": "2026-10-24T08:13:46.683Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.729798, "accuracy_std_dev": 0.06649999628571418, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.12289258193041527, "avg_duration_per_task_ms": 609051.913580247, "avg_output_tokens_per_task": 36908.911335578, "last_run_timestamp": "2026-10-24T16:25:38.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7294042813688214, "accuracy_std_dev": 0.0505548117787454, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.3084836787072243, "avg_duration_per_task_ms": 1347611.5551330799, "avg_output_tokens_per_task": 97858.03675538657, "last_run_timestamp": "2026-10-24T16:28:00.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7293771250000001, "accuracy_std_dev": 0.010428876514724617, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00597866792929293, "avg_duration_per_task_ms": 411352.74242424243, "avg_output_tokens_per_task": 7899.722222222223, "last_run_timestamp": "2026-10-24T16:07:46.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7281143333333333, "accuracy_std_dev": 0.016544932312409815, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004527595117845118, "avg_duration_per_task_ms": 163776.3367003367, "avg_output_tokens_per_task": 7680.468013468014, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7277907692307694, "accuracy_std_dev": 0.009063148715488058, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.19022068764568764, "avg_duration_per_task_ms": 243477.47397047398, "avg_output_tokens_per_task": 37853.29875679876, "last_run_timestamp": "2026-10-24T08:26:35.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7271808509532062, "accuracy_std_dev": 0.03540222199908109, "run_count": 6, "total_questions": 1154, "avg_cost_per_task_usd": 0.011152869150779895, "avg_duration_per_task_ms": 504501.5831889082, "avg_output_tokens_per_task": 55506.62998266898, "last_run_timestamp": "2026-10-10T08:24:27.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7265246666666667, "accuracy_std_dev": 0.015018083728921232, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005456846240179574, "avg_duration_per_task_ms": 293364.7452300786, "avg_output_tokens_per_task": 7944.652076318743, "last_run_timestamp": "2026-10-24T16:01:05.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7265245555555556, "accuracy_std_dev": 0.023902048992182463, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0058103249158249146, "avg_duration_per_task_ms": 345990.65993265994, "avg_output_tokens_per_task": 18519.231200897866, "last_run_timestamp": "2026-10-24T16:02:47.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7264923098256736, "accuracy_std_dev": 0.012443953888007772, "run_count": 13, "total_questions": 2524, "avg_cost_per_task_usd": 0.18690665134706813, "avg_duration_per_task_ms": 345005.80625990493, "avg_output_tokens_per_task": 37190.982171156895, "last_run_timestamp": "2026-10-24T08:44:41.412Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7261503333333333, "accuracy_std_dev": 0.004418546090690373, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.059926168350168355, "avg_duration_per_task_ms": 123042.70033670033, "avg_output_tokens_per_task": 23940.483164983165, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7221381, "accuracy_std_dev": 0.010360325819683453, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.004400843939393938, "avg_duration_per_task_ms": 655428.28989899, "avg_output_tokens_per_task": 7876.879292929293, "last_run_timestamp": "2026-10-24T16:01:45.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.721866038034188, "accuracy_std_dev": 0.03876582276336764, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.010606732051282052, "avg_duration_per_task_ms": 258545.00683760684, "avg_output_tokens_per_task": 48059.278205128205, "last_run_timestamp": "2026-10-24T08:11:11.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.72179058974359, "accuracy_std_dev": 0.013298167116268404, "run_count": 39, "total_questions": 7722, "avg_cost_per_task_usd": 0.1972910025900026, "avg_duration_per_task_ms": 268583.0019425019, "avg_output_tokens_per_task": 36795.75433825434, "last_run_timestamp": "2026-10-24T09:06:24.922Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7208904075342465, "accuracy_std_dev": 0.09870737973249176, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.13571237785388127, "avg_duration_per_task_ms": 977847.7089041095, "avg_output_tokens_per_task": 72379.5450913242, "last_run_timestamp": "2026-10-24T16:26:47.558Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.719697, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021745989898989897, "avg_duration_per_task_ms": 204822.55555555556, "avg_output_tokens_per_task": 77279.37878787878, "last_run_timestamp": "2026-10-24T08:14:03.073Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7174523333333331, "accuracy_std_dev": 0.013163488932987655, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.018911546296296298, "avg_duration_per_task_ms": 531218.9570707071, "avg_output_tokens_per_task": 67191.76851851853, "last_run_timestamp": "2026-10-24T09:17:16.298Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.7163299999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005710287878787878, "avg_duration_per_task_ms": 197152.55555555556, "avg_output_tokens_per_task": 6940.146464646465, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7135241666666668, "accuracy_std_dev": 0.005271695534223852, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015626640572390573, "avg_duration_per_task_ms": 96955.32070707071, "avg_output_tokens_per_task": 10336.517676767677, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7135240000000002, "accuracy_std_dev": 0.009836496361340617, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007830127104377103, "avg_duration_per_task_ms": 79042.44696969698, "avg_output_tokens_per_task": 9942.877946127946, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.712284535775862, "accuracy_std_dev": 0.11489540719980705, "run_count": 13, "total_questions": 2320, "avg_cost_per_task_usd": 0.032706028879310345, "avg_duration_per_task_ms": 1027452.9099137931, "avg_output_tokens_per_task": 34342.02112068966, "last_run_timestamp": "2026-10-24T16:14:06.772Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7121211666666666, "accuracy_std_dev": 0.009066017803068557, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011008058080808082, "avg_duration_per_task_ms": 95816.33922558922, "avg_output_tokens_per_task": 10395.001683501683, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.71170025, "accuracy_std_dev": 0.018456257615982147, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0649322058080808, "avg_duration_per_task_ms": 1066951.6603535353, "avg_output_tokens_per_task": 21770.531565656565, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7112403023255814, "accuracy_std_dev": 0.09623599999999999, "run_count": 2, "total_questions": 344, "avg_cost_per_task_usd": 0.06576875, "avg_duration_per_task_ms": 744084.4970930233, "avg_output_tokens_per_task": 27095.636627906977, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.7107745999999999, "accuracy_std_dev": 0.05091182812510271, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006678358585858586, "avg_duration_per_task_ms": 125158.29696969697, "avg_output_tokens_per_task": 7409.80808080808, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.7106481250000001, "accuracy_std_dev": 0.043384424386631835, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007522804924242423, "avg_duration_per_task_ms": 781654.2998737374, "avg_output_tokens_per_task": 26137.825126262625, "last_run_timestamp": "2026-10-24T16:14:06.772Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7100635555555554, "accuracy_std_dev": 0.054318885093514825, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006609520763187431, "avg_duration_per_task_ms": 110734.31762065095, "avg_output_tokens_per_task": 16252.387766554433, "last_run_timestamp": "2026-10-24T16:28:53.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.709650171648164, "accuracy_std_dev": 0.01649939376664086, "run_count": 6, "total_questions": 1171, "avg_cost_per_task_usd": 0.010711521776259607, "avg_duration_per_task_ms": 103546.38514090521, "avg_output_tokens_per_task": 9904.411614005125, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7096191251428572, "accuracy_std_dev": 0.04936102018827993, "run_count": 18, "total_questions": 3500, "avg_cost_per_task_usd": 0.007894835428571428, "avg_duration_per_task_ms": 520799.64857142855, "avg_output_tokens_per_task": 25638.9, "last_run_timestamp": "2026-10-24T16:12:04.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7093155, "accuracy_std_dev": 0.008890530969332882, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008779646464646466, "avg_duration_per_task_ms": 132997.85521885523, "avg_output_tokens_per_task": 5690.617845117845, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.707752053820598, "accuracy_std_dev": 0.25911579823070297, "run_count": 8, "total_questions": 1505, "avg_cost_per_task_usd": 0.025081889036544854, "avg_duration_per_task_ms": 430774.81063122925, "avg_output_tokens_per_task": 14989.863122923589, "last_run_timestamp": "2026-10-17T16:03:44.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.705263, "accuracy_std_dev": null, "run_count": 1, "total_questions": 95, "avg_cost_per_task_usd": 0.10938622105263157, "avg_duration_per_task_ms": 1384847.8, "avg_output_tokens_per_task": 40599.36842105263, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7048612500000001, "accuracy_std_dev": 0.04887384717246537, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.016058294823232323, "avg_duration_per_task_ms": 999836.755050505, "avg_output_tokens_per_task": 49417.84027777778, "last_run_timestamp": "2026-10-24T16:05:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7045453333333335, "accuracy_std_dev": 0.009510928024587757, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00597795707070707, "avg_duration_per_task_ms": 110020.64814814815, "avg_output_tokens_per_task": 5546.97138047138, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7032012317327765, "accuracy_std_dev": 0.20342432055346774, "run_count": 5, "total_questions": 958, "avg_cost_per_task_usd": 0.08802485803757831, "avg_duration_per_task_ms": 1349048.8903966597, "avg_output_tokens_per_task": 33413.259916492694, "last_run_timestamp": "2026-10-24T16:13:05.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7031426666666668, "accuracy_std_dev": 0.014373068248947025, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011147152356902357, "avg_duration_per_task_ms": 198021.7255892256, "avg_output_tokens_per_task": 5803.353535353535, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.7030114672897196, "accuracy_std_dev": 0.02694021764221664, "run_count": 10, "total_questions": 1926, "avg_cost_per_task_usd": 0.009145764278296989, "avg_duration_per_task_ms": 363422.58099688473, "avg_output_tokens_per_task": 44741.07476635514, "last_run_timestamp": "2026-10-24T08:01:41.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7027678642599278, "accuracy_std_dev": 0.009533289144800297, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.010358495306859207, "avg_duration_per_task_ms": 145686.42599277978, "avg_output_tokens_per_task": 9350.431768953069, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.7011786666666667, "accuracy_std_dev": 0.0268613535999626, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.010966026094276094, "avg_duration_per_task_ms": 64637.51430976431, "avg_output_tokens_per_task": 9084.513468013469, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6995251282660333, "accuracy_std_dev": 0.037727051631422225, "run_count": 5, "total_questions": 842, "avg_cost_per_task_usd": 0.03918485154394299, "avg_duration_per_task_ms": 785269.9073634205, "avg_output_tokens_per_task": 41981.14845605701, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6938431428571429, "accuracy_std_dev": 0.009541538965844799, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005521870129870129, "avg_duration_per_task_ms": 387910.27056277054, "avg_output_tokens_per_task": 6090.331168831169, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6932966363636364, "accuracy_std_dev": 0.13147150091889792, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.609068142332415, "avg_duration_per_task_ms": 632474.5367309458, "avg_output_tokens_per_task": 40521.938016528926, "last_run_timestamp": "2026-10-24T08:08:54.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6932378040665435, "accuracy_std_dev": 0.046335214744527585, "run_count": 11, "total_questions": 2164, "avg_cost_per_task_usd": 0.0074276700554528645, "avg_duration_per_task_ms": 281567.51894639555, "avg_output_tokens_per_task": 40195.87939001848, "last_run_timestamp": "2026-10-24T08:42:05.963Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.6927610000000001, "accuracy_std_dev": 0.01993714402817014, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.009704229797979799, "avg_duration_per_task_ms": 259526.76094276094, "avg_output_tokens_per_task": 6456.316498316498, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.6899941197301855, "accuracy_std_dev": 0.005866752821337055, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.0068884283305227665, "avg_duration_per_task_ms": 1677214.3709949409, "avg_output_tokens_per_task": 49011.02360876897, "last_run_timestamp": "2026-10-24T16:08:06.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6855188289251547, "accuracy_std_dev": 0.09527891619753961, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.011474473832301631, "avg_duration_per_task_ms": 906865.2909397861, "avg_output_tokens_per_task": 33586.68429938098, "last_run_timestamp": "2026-10-24T16:17:39.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6844864918238994, "accuracy_std_dev": 0.03067807249818805, "run_count": 7, "total_questions": 795, "avg_cost_per_task_usd": 0.00916637610062893, "avg_duration_per_task_ms": 54426.96603773585, "avg_output_tokens_per_task": 5936.947169811321, "last_run_timestamp": "2026-10-24T16:45:45.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6844233356431357, "accuracy_std_dev": 0.1414237811051766, "run_count": 35, "total_questions": 6927, "avg_cost_per_task_usd": 0.6621923740435977, "avg_duration_per_task_ms": 602080.901255955, "avg_output_tokens_per_task": 41272.465136422696, "last_run_timestamp": "2026-10-24T08:43:07.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.6830809999999999, "accuracy_std_dev": 0.14005659745795623, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.6164722958754209, "avg_duration_per_task_ms": 635625.3400673401, "avg_output_tokens_per_task": 40941.11069023569, "last_run_timestamp": "2026-10-24T08:26:35.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6825664722222222, "accuracy_std_dev": 0.136152835465126, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.6585870970819304, "avg_duration_per_task_ms": 454569.8709315376, "avg_output_tokens_per_task": 40985.43560606061, "last_run_timestamp": "2026-10-24T08:24:10.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.6807458630136987, "accuracy_std_dev": 0.019503295687310546, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.0021889891552511415, "avg_duration_per_task_ms": 195459.59874429225, "avg_output_tokens_per_task": 4065.609589041096, "last_run_timestamp": "2026-10-24T16:39:13.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.6805556666666668, "accuracy_std_dev": 0.14593486187813, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.6194684494949494, "avg_duration_per_task_ms": 636905.944023569, "avg_output_tokens_per_task": 41202.27062289562, "last_run_timestamp": "2026-10-24T08:14:03.137Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.6805556666666668, "accuracy_std_dev": 0.06224365077839043, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007968234006734005, "avg_duration_per_task_ms": 1481788.1506734006, "avg_output_tokens_per_task": 62936.50420875421, "last_run_timestamp": "2026-10-24T16:26:47.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6797738037518039, "accuracy_std_dev": 0.015747258010206083, "run_count": 10, "total_questions": 1386, "avg_cost_per_task_usd": 0.004640227272727273, "avg_duration_per_task_ms": 817645.9841269841, "avg_output_tokens_per_task": 46248.668831168834, "last_run_timestamp": "2026-10-24T16:05:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.6748038333333333, "accuracy_std_dev": 0.04521916160735648, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01522573063973064, "avg_duration_per_task_ms": 286760.11026936024, "avg_output_tokens_per_task": 8293.19696969697, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6746166405451448, "accuracy_std_dev": 0.03659361862233009, "run_count": 9, "total_questions": 1761, "avg_cost_per_task_usd": 0.027729125496876776, "avg_duration_per_task_ms": 864050.6172629189, "avg_output_tokens_per_task": 18372.465644520158, "last_run_timestamp": "2026-10-24T16:22:30.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6735395649484536, "accuracy_std_dev": 0.02076995482325371, "run_count": 5, "total_questions": 485, "avg_cost_per_task_usd": 0.003951113402061856, "avg_duration_per_task_ms": 479082.6536082474, "avg_output_tokens_per_task": 26380.544329896908, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.66813975, "accuracy_std_dev": 0.0169491097609137, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005768640151515151, "avg_duration_per_task_ms": 418288.6957070707, "avg_output_tokens_per_task": 15810.410353535353, "last_run_timestamp": "2026-10-24T16:11:51.309Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6662457499999999, "accuracy_std_dev": 0.017551141520923935, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005718719696969698, "avg_duration_per_task_ms": 566951.7815656565, "avg_output_tokens_per_task": 16218.064393939394, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6661054999999999, "accuracy_std_dev": 0.006849761936252494, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002126188552188552, "avg_duration_per_task_ms": 243791.89646464647, "avg_output_tokens_per_task": 3027.095117845118, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6656563999999999, "accuracy_std_dev": 0.00579280583482652, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0027868222222222224, "avg_duration_per_task_ms": 87457.03737373737, "avg_output_tokens_per_task": 2951.019191919192, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6651514999999998, "accuracy_std_dev": 0.008019320822239275, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.005088861616161617, "avg_duration_per_task_ms": 104484.55757575757, "avg_output_tokens_per_task": 9583.160606060606, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6649444988518944, "accuracy_std_dev": 0.01638003285131017, "run_count": 10, "total_questions": 1742, "avg_cost_per_task_usd": 0.006743279563719862, "avg_duration_per_task_ms": 475467.2680826636, "avg_output_tokens_per_task": 47990.47588978186, "last_run_timestamp": "2026-10-24T16:02:28.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6637718184210527, "accuracy_std_dev": 0.20207857374716895, "run_count": 10, "total_questions": 1900, "avg_cost_per_task_usd": 0.012164477368421052, "avg_duration_per_task_ms": 311613.41736842104, "avg_output_tokens_per_task": 14454.243684210527, "last_run_timestamp": "2026-10-17T08:17:57.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.66372075, "accuracy_std_dev": 0.00844903779654822, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00248682702020202, "avg_duration_per_task_ms": 87766.55681818182, "avg_output_tokens_per_task": 2304.1906565656564, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.6619902222222221, "accuracy_std_dev": 0.13296183010404633, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.014990915263748598, "avg_duration_per_task_ms": 410516.86419753084, "avg_output_tokens_per_task": 13130.794051627385, "last_run_timestamp": "2026-10-24T16:02:06.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6607744999999999, "accuracy_std_dev": 0.011572094721786573, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015565636363636365, "avg_duration_per_task_ms": 14159.460858585859, "avg_output_tokens_per_task": 1751.3977272727273, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6601731428571429, "accuracy_std_dev": 0.023160494075833857, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008067838383838384, "avg_duration_per_task_ms": 808632.0974025974, "avg_output_tokens_per_task": 55153.242424242424, "last_run_timestamp": "2026-10-24T16:14:24.977Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6594651666666667, "accuracy_std_dev": 0.015979149081259345, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.008953686026936029, "avg_duration_per_task_ms": 317080.76767676766, "avg_output_tokens_per_task": 59355.69332210999, "last_run_timestamp": "2026-10-24T16:22:30.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6570467857142857, "accuracy_std_dev": 0.14261200486794257, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.6177520173160173, "avg_duration_per_task_ms": 633109.20995671, "avg_output_tokens_per_task": 41054.18795093795, "last_run_timestamp": "2026-10-24T08:12:02.408Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6567339999999999, "accuracy_std_dev": 0.02088421366487137, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007585557575757577, "avg_duration_per_task_ms": 623991.9545454546, "avg_output_tokens_per_task": 58619.2005050505, "last_run_timestamp": "2026-10-24T16:15:47.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6552547006369427, "accuracy_std_dev": 0.08012703749983525, "run_count": 4, "total_questions": 628, "avg_cost_per_task_usd": 0.06530196178343949, "avg_duration_per_task_ms": 1671084.4410828026, "avg_output_tokens_per_task": 24249.23566878981, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6532027662178702, "accuracy_std_dev": 0.21133166802603598, "run_count": 9, "total_questions": 1634, "avg_cost_per_task_usd": 0.008997689106487149, "avg_duration_per_task_ms": 186886.9828641371, "avg_output_tokens_per_task": 14831.725826193391, "last_run_timestamp": "2026-10-24T16:26:15.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6529368014874142, "accuracy_std_dev": 0.054041396139361146, "run_count": 9, "total_questions": 1748, "avg_cost_per_task_usd": 0.05780651773455378, "avg_duration_per_task_ms": 1127057.8415331808, "avg_output_tokens_per_task": 24934.28718535469, "last_run_timestamp": "2026-10-24T16:10:43.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.651601, "accuracy_std_dev": null, "run_count": 1, "total_questions": 177, "avg_cost_per_task_usd": 0.03645816384180791, "avg_duration_per_task_ms": 1143037.5875706214, "avg_output_tokens_per_task": 29354.60451977401, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.651515, "accuracy_std_dev": 0.1905456270048375, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.058297676767676776, "avg_duration_per_task_ms": 1098090.5791245792, "avg_output_tokens_per_task": 23174.20202020202, "last_run_timestamp": "2026-10-17T16:03:44.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.6510548691983122, "accuracy_std_dev": 0.12013379799957409, "run_count": 9, "total_questions": 1185, "avg_cost_per_task_usd": 0.0951064641350211, "avg_duration_per_task_ms": 1230349.5139240506, "avg_output_tokens_per_task": 27086.752742616034, "last_run_timestamp": "2026-10-24T16:06:30.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6505050000000001, "accuracy_std_dev": 0.008356827340564098, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003142527272727273, "avg_duration_per_task_ms": 40777.49797979798, "avg_output_tokens_per_task": 2977.0545454545454, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.011195005920300148, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004075954545454546, "avg_duration_per_task_ms": 28668.510101010103, "avg_output_tokens_per_task": 2354.649831649832, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.6488281909758657, "accuracy_std_dev": 0.006868775133893942, "run_count": 5, "total_questions": 953, "avg_cost_per_task_usd": 0.003697750262329486, "avg_duration_per_task_ms": 59831.95383001049, "avg_output_tokens_per_task": 2242.5666316894017, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6478116, "accuracy_std_dev": 0.01301437818107341, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0023702696969696967, "avg_duration_per_task_ms": 65368.43434343435, "avg_output_tokens_per_task": 2772.5838383838386, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6473065000000001, "accuracy_std_dev": 0.009707729665065862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00413114393939394, "avg_duration_per_task_ms": 27377.72601010101, "avg_output_tokens_per_task": 2388.9305555555557, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6470586811145511, "accuracy_std_dev": 0.06899576470516142, "run_count": 4, "total_questions": 646, "avg_cost_per_task_usd": 0.006973526315789473, "avg_duration_per_task_ms": 477220.6222910217, "avg_output_tokens_per_task": 36135.41950464396, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6464646666666667, "accuracy_std_dev": 0.013537798302858865, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015368191919191919, "avg_duration_per_task_ms": 14060.520202020201, "avg_output_tokens_per_task": 1727.3535353535353, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.6454544, "accuracy_std_dev": 0.002519632084253531, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.002477851515151515, "avg_duration_per_task_ms": 59827.33737373738, "avg_output_tokens_per_task": 2673.8050505050505, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6451553333333333, "accuracy_std_dev": 0.022132801022715396, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007395829966329966, "avg_duration_per_task_ms": 612621.9057239058, "avg_output_tokens_per_task": 44969.978114478115, "last_run_timestamp": "2026-10-24T16:13:05.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.6439055298507463, "accuracy_std_dev": 0.20314360210271282, "run_count": 7, "total_questions": 1340, "avg_cost_per_task_usd": 0.07648400820895522, "avg_duration_per_task_ms": 1782783.4514925373, "avg_output_tokens_per_task": 36935.7671641791, "last_run_timestamp": "2026-10-24T16:17:39.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.643659, "accuracy_std_dev": 0.015752669318774726, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015761657407407408, "avg_duration_per_task_ms": 18961.11531986532, "avg_output_tokens_per_task": 1683.8872053872053, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6428171666666667, "accuracy_std_dev": 0.012039053318494584, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.002693207070707071, "avg_duration_per_task_ms": 142395.0521885522, "avg_output_tokens_per_task": 9633.525252525253, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.6419754999999999, "accuracy_std_dev": 0.013802963036367705, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004449612794612795, "avg_duration_per_task_ms": 74990.68265993266, "avg_output_tokens_per_task": 8364.945286195287, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6419754444444444, "accuracy_std_dev": 0.014018014458475362, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007092590347923681, "avg_duration_per_task_ms": 241593.06509539843, "avg_output_tokens_per_task": 11377.103254769921, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6405723333333334, "accuracy_std_dev": 0.0030349496792460334, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004210166666666667, "avg_duration_per_task_ms": 30431.28367003367, "avg_output_tokens_per_task": 2308.9638047138046, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6399558039735098, "accuracy_std_dev": 0.01859859452183148, "run_count": 8, "total_questions": 1510, "avg_cost_per_task_usd": 0.005625637086092715, "avg_duration_per_task_ms": 835153.6013245033, "avg_output_tokens_per_task": 34992.65165562914, "last_run_timestamp": "2026-10-24T08:14:51.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6368495215189873, "accuracy_std_dev": 0.014462691735980549, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0026009772151898737, "avg_duration_per_task_ms": 238051.80337552744, "avg_output_tokens_per_task": 8445.996624472575, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6353816666666667, "accuracy_std_dev": 0.011788772294951762, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010992390572390571, "avg_duration_per_task_ms": 380904.83164983167, "avg_output_tokens_per_task": 11589.25, "last_run_timestamp": "2026-10-24T16:24:45.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6342374433962265, "accuracy_std_dev": 0.011539125582122754, "run_count": 5, "total_questions": 848, "avg_cost_per_task_usd": 0.007657488207547171, "avg_duration_per_task_ms": 34742.110849056604, "avg_output_tokens_per_task": 12687.265330188678, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6334780000000001, "accuracy_std_dev": 0.013155152146592608, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0016263744588744587, "avg_duration_per_task_ms": 75470.49134199134, "avg_output_tokens_per_task": 8221.989177489177, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6325426993295513, "accuracy_std_dev": 0.010534965676735725, "run_count": 10, "total_questions": 1939, "avg_cost_per_task_usd": 0.01766013512119649, "avg_duration_per_task_ms": 478451.96183599794, "avg_output_tokens_per_task": 88282.73388344508, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6310882796803652, "accuracy_std_dev": 0.015991023651789744, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.01677861757990868, "avg_duration_per_task_ms": 87974.94634703196, "avg_output_tokens_per_task": 55697.56792237443, "last_run_timestamp": "2026-10-24T16:45:19.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6306148156028369, "accuracy_std_dev": 0.05281625698168657, "run_count": 3, "total_questions": 282, "avg_cost_per_task_usd": 0.0037584255319148938, "avg_duration_per_task_ms": 208452.59574468085, "avg_output_tokens_per_task": 12308.468085106382, "last_run_timestamp": "2026-10-23T09:29:14.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6299665, "accuracy_std_dev": 0.033651413370763474, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0068110621212121225, "avg_duration_per_task_ms": 109012.8005050505, "avg_output_tokens_per_task": 9497.993434343434, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6289855826086956, "accuracy_std_dev": 0.01629778854411147, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.0038922746376811594, "avg_duration_per_task_ms": 225279.24782608697, "avg_output_tokens_per_task": 12705.184057971015, "last_run_timestamp": "2026-10-23T08:08:11.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6285795064570466, "accuracy_std_dev": 0.020919173358428876, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01718929421673217, "avg_duration_per_task_ms": 631165.4817518248, "avg_output_tokens_per_task": 83912.1010668164, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6280247450759707, "accuracy_std_dev": 0.010380266086410858, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.0018582673044456948, "avg_duration_per_task_ms": 228947.18570624647, "avg_output_tokens_per_task": 10187.530106921778, "last_run_timestamp": "2026-10-24T16:03:25.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.627104375, "accuracy_std_dev": 0.008333089267155095, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024497946969696968, "avg_duration_per_task_ms": 1060409.1414141415, "avg_output_tokens_per_task": 101874.34027777778, "last_run_timestamp": "2026-10-23T09:03:34.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6182949425287356, "accuracy_std_dev": 0.0013475000000000015, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.011105244252873564, "avg_duration_per_task_ms": 345775.4454022989, "avg_output_tokens_per_task": 38606.68103448276, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6170940030769231, "accuracy_std_dev": 0.014320214565431623, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.015717948717948717, "avg_duration_per_task_ms": 780521.6543589743, "avg_output_tokens_per_task": 16712.84717948718, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6128760749682338, "accuracy_std_dev": 0.007225322047320813, "run_count": 4, "total_questions": 787, "avg_cost_per_task_usd": 0.0033133367217280814, "avg_duration_per_task_ms": 404332.3011435832, "avg_output_tokens_per_task": 13398.312579415502, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.6121724426966293, "accuracy_std_dev": 0.10417909241135824, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.009812219101123598, "avg_duration_per_task_ms": 327793.9264044944, "avg_output_tokens_per_task": 27904.43820224719, "last_run_timestamp": "2026-10-24T16:05:40.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6099888333333333, "accuracy_std_dev": 0.01106802965627677, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003627024410774411, "avg_duration_per_task_ms": 33220.15151515151, "avg_output_tokens_per_task": 2140.8367003367002, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6099860008438818, "accuracy_std_dev": 0.004922906311880752, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0040446168776371305, "avg_duration_per_task_ms": 27156.491983122363, "avg_output_tokens_per_task": 2148.8616033755275, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007801348484848485, "avg_duration_per_task_ms": 208074.1717171717, "avg_output_tokens_per_task": 15081.5, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.608586, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007727368686868687, "avg_duration_per_task_ms": 263769.8611111111, "avg_output_tokens_per_task": 15259.444444444445, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.6080220220713072, "accuracy_std_dev": 0.18476388731221863, "run_count": 12, "total_questions": 2356, "avg_cost_per_task_usd": 0.0260303675721562, "avg_duration_per_task_ms": 717454.6184210526, "avg_output_tokens_per_task": 36816.12648556876, "last_run_timestamp": "2026-10-24T16:13:05.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.607708238128011, "accuracy_std_dev": 0.09072663406949609, "run_count": 9, "total_questions": 1453, "avg_cost_per_task_usd": 0.05455010392291811, "avg_duration_per_task_ms": 329922.3234686855, "avg_output_tokens_per_task": 20685.869924294562, "last_run_timestamp": "2026-10-24T16:28:50.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6062142522796352, "accuracy_std_dev": 0.009864001918085761, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.008205017223910841, "avg_duration_per_task_ms": 297243.1023302938, "avg_output_tokens_per_task": 17225.689969604864, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Mara", "accuracy": 0.6061323018867925, "accuracy_std_dev": 0.029270000000000018, "run_count": 2, "total_questions": 212, "avg_cost_per_task_usd": 0.034763094339622644, "avg_duration_per_task_ms": 71660.68867924529, "avg_output_tokens_per_task": 7461.462264150943, "last_run_timestamp": "2026-10-17T16:06:00.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6059645129698252, "accuracy_std_dev": 0.012733602453351516, "run_count": 10, "total_questions": 1889, "avg_cost_per_task_usd": 0.008509499205929062, "avg_duration_per_task_ms": 554697.4531498147, "avg_output_tokens_per_task": 13980.221281101112, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6050506, "accuracy_std_dev": 0.015451550117706637, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005770865656565657, "avg_duration_per_task_ms": 636718.6202020202, "avg_output_tokens_per_task": 19268.987878787877, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.604377, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00791619191919192, "avg_duration_per_task_ms": 146373.4797979798, "avg_output_tokens_per_task": 15059.80303030303, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6042256518448439, "accuracy_std_dev": 0.02010368015853481, "run_count": 6, "total_questions": 1057, "avg_cost_per_task_usd": 0.0021907710501419112, "avg_duration_per_task_ms": 51585.36707663198, "avg_output_tokens_per_task": 2058.464522232734, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6038161666666666, "accuracy_std_dev": 0.007251665174672309, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0035006599326599325, "avg_duration_per_task_ms": 56534.7601010101, "avg_output_tokens_per_task": 2119.1574074074074, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6035354999999999, "accuracy_std_dev": 0.0052117790868377945, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.010990611952861952, "avg_duration_per_task_ms": 32954.444444444445, "avg_output_tokens_per_task": 2093.7045454545455, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.602339, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0316934269005848, "avg_duration_per_task_ms": 1131594.5263157894, "avg_output_tokens_per_task": 37899.070175438595, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6017236343772759, "accuracy_std_dev": 0.07577236111713366, "run_count": 7, "total_questions": 1373, "avg_cost_per_task_usd": 0.012778518572469047, "avg_duration_per_task_ms": 1389201.3306627823, "avg_output_tokens_per_task": 56525.64020393299, "last_run_timestamp": "2026-10-24T16:33:52.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.6016879113924051, "accuracy_std_dev": 0.05594315696758628, "run_count": 4, "total_questions": 790, "avg_cost_per_task_usd": 0.033461643037974685, "avg_duration_per_task_ms": 2683325.098734177, "avg_output_tokens_per_task": 25317.36075949367, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.599327, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014991068181818182, "avg_duration_per_task_ms": 284433.01515151514, "avg_output_tokens_per_task": 12302.075757575758, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5973626666666666, "accuracy_std_dev": 0.014397638598819675, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003072445286195286, "avg_duration_per_task_ms": 655562.0867003367, "avg_output_tokens_per_task": 13595.005050505051, "last_run_timestamp": "2026-10-07T13:51:26.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.5962405000000001, "accuracy_std_dev": 0.005702057428390789, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003268709595959596, "avg_duration_per_task_ms": 65099.85437710438, "avg_output_tokens_per_task": 2080.245791245791, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5948373333333333, "accuracy_std_dev": 0.010316724200162651, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04306378282828283, "avg_duration_per_task_ms": 401108.6296296296, "avg_output_tokens_per_task": 21434.68181818182, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5942281016850668, "accuracy_std_dev": 0.00965016140081236, "run_count": 9, "total_questions": 1721, "avg_cost_per_task_usd": 0.009055742010459035, "avg_duration_per_task_ms": 523063.0418361418, "avg_output_tokens_per_task": 45085.088901801275, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.591751, "accuracy_std_dev": 0.006627755628164546, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017204867003367003, "avg_duration_per_task_ms": 118615.4696969697, "avg_output_tokens_per_task": 42873.59259259259, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5906884318766067, "accuracy_std_dev": 0.010206288841526426, "run_count": 6, "total_questions": 1167, "avg_cost_per_task_usd": 0.006399965724078834, "avg_duration_per_task_ms": 683293.4078834619, "avg_output_tokens_per_task": 42407.72150814053, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5905352222222223, "accuracy_std_dev": 0.008575970638654383, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.010281984287317622, "avg_duration_per_task_ms": 11349.746913580248, "avg_output_tokens_per_task": 13254.841189674524, "last_run_timestamp": "2026-10-24T16:20:41.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5841753333333334, "accuracy_std_dev": 0.033949349496304376, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02777477777777778, "avg_duration_per_task_ms": 1412449.8375420875, "avg_output_tokens_per_task": 69325.78367003366, "last_run_timestamp": "2026-10-13T21:28:41.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.5816397652439025, "accuracy_std_dev": 0.007030422274657478, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.025769568089430896, "avg_duration_per_task_ms": 1126074.3231707318, "avg_output_tokens_per_task": 67814.54166666667, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.5774414999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010289136363636365, "avg_duration_per_task_ms": 331432.74242424243, "avg_output_tokens_per_task": 40883.856060606064, "last_run_timestamp": "2026-10-17T16:06:05.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.575269, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.03680247311827957, "avg_duration_per_task_ms": 1058115.6344086023, "avg_output_tokens_per_task": 47809.01612903226, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5710451276332094, "accuracy_std_dev": 0.03546539526778179, "run_count": 8, "total_questions": 807, "avg_cost_per_task_usd": 0.03254549814126394, "avg_duration_per_task_ms": 62701.02726146221, "avg_output_tokens_per_task": 6973.700123915737, "last_run_timestamp": "2026-10-24T16:01:29.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.5683499999999999, "accuracy_std_dev": 0.1383388370104361, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0030159525252525248, "avg_duration_per_task_ms": 227529.94494949494, "avg_output_tokens_per_task": 7208.462626262626, "last_run_timestamp": "2026-10-24T16:03:25.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5621250282776349, "accuracy_std_dev": 0.06329693423460571, "run_count": 4, "total_questions": 389, "avg_cost_per_task_usd": 0.04400786118251928, "avg_duration_per_task_ms": 513471.6503856041, "avg_output_tokens_per_task": 21046.47557840617, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5572391666666667, "accuracy_std_dev": 0.009959776661094785, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004621723905723906, "avg_duration_per_task_ms": 448955.2752525252, "avg_output_tokens_per_task": 16279.704545454546, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.5512932188317349, "accuracy_std_dev": 0.02565312085410966, "run_count": 6, "total_questions": 1147, "avg_cost_per_task_usd": 0.005740845684394072, "avg_duration_per_task_ms": 730028.2885789014, "avg_output_tokens_per_task": 14883.85701830863, "last_run_timestamp": "2026-10-13T09:46:38.737Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5505053333333333, "accuracy_std_dev": 0.016890890559378122, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.017397131313131312, "avg_duration_per_task_ms": 129348.1271043771, "avg_output_tokens_per_task": 43400.656565656565, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5488216666666668, "accuracy_std_dev": 0.006298902038362641, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0171891531986532, "avg_duration_per_task_ms": 121387.12121212122, "avg_output_tokens_per_task": 42892.78114478115, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.535985, "accuracy_std_dev": 0.011835000612589788, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0018682651515151514, "avg_duration_per_task_ms": 92043.92676767676, "avg_output_tokens_per_task": 1839.9911616161617, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.53367, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00486480303030303, "avg_duration_per_task_ms": 109732.54545454546, "avg_output_tokens_per_task": 10482.964646464647, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.5331086666666667, "accuracy_std_dev": 0.05754091454106565, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005119237373737374, "avg_duration_per_task_ms": 101325.84511784512, "avg_output_tokens_per_task": 5420.193602693603, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.532164, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.03933312280701755, "avg_duration_per_task_ms": 1068622.9473684211, "avg_output_tokens_per_task": 51714.59064327485, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5238096808510638, "accuracy_std_dev": 0.00755476843854264, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0166975390070922, "avg_duration_per_task_ms": 736031.547112462, "avg_output_tokens_per_task": 42415.456940222895, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5230117451776649, "accuracy_std_dev": 0.08115411707264149, "run_count": 5, "total_questions": 985, "avg_cost_per_task_usd": 0.0036350548223350256, "avg_duration_per_task_ms": 131062.38578680203, "avg_output_tokens_per_task": 3887.226395939086, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.522126, "accuracy_std_dev": 0.02474951208754282, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01514130735930736, "avg_duration_per_task_ms": 13491.472582972583, "avg_output_tokens_per_task": 1320.8239538239538, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.513468, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0047852878787878785, "avg_duration_per_task_ms": 118138.42929292929, "avg_output_tokens_per_task": 10671.39898989899, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.506823, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.03426515789473684, "avg_duration_per_task_ms": 1056421.3684210526, "avg_output_tokens_per_task": 40494.5730994152, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5065361568627451, "accuracy_std_dev": 0.1358455723807, "run_count": 5, "total_questions": 918, "avg_cost_per_task_usd": 0.004705715686274509, "avg_duration_per_task_ms": 112349.47167755991, "avg_output_tokens_per_task": 4227.696078431372, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.5056116666666668, "accuracy_std_dev": 0.007570922855827376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012942734006734005, "avg_duration_per_task_ms": 12612.308080808081, "avg_output_tokens_per_task": 1100.3198653198654, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5, "accuracy_std_dev": 0.015120700182200542, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008362104377104377, "avg_duration_per_task_ms": 19874.548821548822, "avg_output_tokens_per_task": 1898.5521885521885, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.4979423333333333, "accuracy_std_dev": 0.013034980185809425, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0023551829405162737, "avg_duration_per_task_ms": 10287.485970819303, "avg_output_tokens_per_task": 2334.3832772166106, "last_run_timestamp": "2026-10-24T16:02:08.875Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.49764320000000006, "accuracy_std_dev": 0.01852187333290023, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0008495191919191919, "avg_duration_per_task_ms": 13033.880808080808, "avg_output_tokens_per_task": 1851.6353535353535, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4966332500000001, "accuracy_std_dev": 0.014773079898501197, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008230315656565656, "avg_duration_per_task_ms": 17739.39898989899, "avg_output_tokens_per_task": 1863.909090909091, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.49197393750000007, "accuracy_std_dev": 0.015797164320219007, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.01112520286885246, "avg_duration_per_task_ms": 456396.28176229505, "avg_output_tokens_per_task": 27747.69262295082, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.48962583163265305, "accuracy_std_dev": 0.02041486033848873, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.011151638775510205, "avg_duration_per_task_ms": 495714.47040816327, "avg_output_tokens_per_task": 27939.333673469388, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.48512888888888883, "accuracy_std_dev": 0.005611738895574061, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.002352908529741863, "avg_duration_per_task_ms": 15334.566217732885, "avg_output_tokens_per_task": 2211.6778900112236, "last_run_timestamp": "2026-10-24T16:21:36.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.4820423333333333, "accuracy_std_dev": 0.009490110618720696, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.002550328282828283, "avg_duration_per_task_ms": 48799.52581369248, "avg_output_tokens_per_task": 2783.7373737373737, "last_run_timestamp": "2026-10-24T16:03:59.174Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.48202820260989004, "accuracy_std_dev": 0.015459058724256144, "run_count": 15, "total_questions": 2912, "avg_cost_per_task_usd": 0.0012292771291208791, "avg_duration_per_task_ms": 58443.023008241755, "avg_output_tokens_per_task": 2247.816620879121, "last_run_timestamp": "2026-10-24T16:04:33.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.48091999999999996, "accuracy_std_dev": 0.004418715424192887, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.048067654882154885, "avg_duration_per_task_ms": 46868.4898989899, "avg_output_tokens_per_task": 16842.27946127946, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4783013333333333, "accuracy_std_dev": 0.014050459771045853, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.001328496071829405, "avg_duration_per_task_ms": 46533.534792368126, "avg_output_tokens_per_task": 2380.4786756453423, "last_run_timestamp": "2026-10-24T16:03:59.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47777759999999997, "accuracy_std_dev": 0.010931021024588695, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0010385141414141412, "avg_duration_per_task_ms": 206806.0606060606, "avg_output_tokens_per_task": 2333.3515151515153, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.47325066666666665, "accuracy_std_dev": 0.009636497738632366, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0014022805836139168, "avg_duration_per_task_ms": 75040.13580246913, "avg_output_tokens_per_task": 2343.6004489337824, "last_run_timestamp": "2026-10-24T16:04:55.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.472642875, "accuracy_std_dev": 0.00832212530603661, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0031359400252525256, "avg_duration_per_task_ms": 59854.52904040404, "avg_output_tokens_per_task": 2209.2967171717173, "last_run_timestamp": "2026-10-17T16:05:55.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.4711285466816648, "accuracy_std_dev": 0.0145633743402015, "run_count": 9, "total_questions": 1778, "avg_cost_per_task_usd": 0.0012865393700787402, "avg_duration_per_task_ms": 87597.63948256469, "avg_output_tokens_per_task": 2944.271653543307, "last_run_timestamp": "2026-10-24T16:02:28.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.4664644357937311, "accuracy_std_dev": 0.0072384874663150465, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.0021660940343781595, "avg_duration_per_task_ms": 35847.60161779575, "avg_output_tokens_per_task": 2625.268958543984, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46456220000000004, "accuracy_std_dev": 0.016514098623903146, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0007998439393939393, "avg_duration_per_task_ms": 160078.42626262625, "avg_output_tokens_per_task": 2248.7131313131313, "last_run_timestamp": "2026-10-24T16:22:30.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.45862900000000006, "accuracy_std_dev": null, "run_count": 1, "total_questions": 141, "avg_cost_per_task_usd": 0.0011888297872340425, "avg_duration_per_task_ms": 278701.0070921986, "avg_output_tokens_per_task": 7719.879432624113, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4320986666666667, "accuracy_std_dev": 0.00781631474687543, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010060690235690235, "avg_duration_per_task_ms": 20918.510101010103, "avg_output_tokens_per_task": 1483.5286195286196, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4315373333333334, "accuracy_std_dev": 0.010588221658469791, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010723358585858589, "avg_duration_per_task_ms": 21765.176767676767, "avg_output_tokens_per_task": 1505.858585858586, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4305555, "accuracy_std_dev": 0.007469952225416168, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010262436868686867, "avg_duration_per_task_ms": 17602.7702020202, "avg_output_tokens_per_task": 1517.9292929292928, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.422559, "accuracy_std_dev": 0.09221941654915557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019149646464646466, "avg_duration_per_task_ms": 3630846.4646464647, "avg_output_tokens_per_task": 17674.203703703704, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.41010556441717794, "accuracy_std_dev": 0.09310346585944049, "run_count": 10, "total_questions": 1956, "avg_cost_per_task_usd": 0.004216284764826176, "avg_duration_per_task_ms": 41336.05981595092, "avg_output_tokens_per_task": 1775.0184049079755, "last_run_timestamp": "2026-10-24T16:02:47.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.38720499999999997, "accuracy_std_dev": 0.010647388881786938, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00015740101010101008, "avg_duration_per_task_ms": 6591.242424242424, "avg_output_tokens_per_task": 937.0363636363636, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.37878799999999996, "accuracy_std_dev": 0.01683499999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000344469696969697, "avg_duration_per_task_ms": 46431.74494949495, "avg_output_tokens_per_task": 2036.358585858586, "last_run_timestamp": "2026-10-07T13:35:26.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.32428450000000003, "accuracy_std_dev": 0.09813129033340996, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008669871843434344, "avg_duration_per_task_ms": 575303.3933080808, "avg_output_tokens_per_task": 21440.64835858586, "last_run_timestamp": "2026-10-24T16:36:16.779Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2891415, "accuracy_std_dev": 0.003232916137174003, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006112626262626263, "avg_duration_per_task_ms": 10017.040404040405, "avg_output_tokens_per_task": 577.7083333333334, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.28554066666666666, "accuracy_std_dev": 0.017098526187299817, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.002770008978675645, "avg_duration_per_task_ms": 51158.87991021325, "avg_output_tokens_per_task": 34564.77048260382, "last_run_timestamp": "2026-10-24T16:02:28.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.28226711111111114, "accuracy_std_dev": 0.020967071598868986, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0013839943883277217, "avg_duration_per_task_ms": 137971.6234567901, "avg_output_tokens_per_task": 19125.452300785633, "last_run_timestamp": "2026-10-24T16:08:06.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2739361560283688, "accuracy_std_dev": 0.008153796102974918, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.00006610283687943263, "avg_duration_per_task_ms": 8879.929078014184, "avg_output_tokens_per_task": 551.2074468085107, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.2735093284457478, "accuracy_std_dev": 0.01586648965587537, "run_count": 9, "total_questions": 1705, "avg_cost_per_task_usd": 0.009129268035190615, "avg_duration_per_task_ms": 373704.25923753664, "avg_output_tokens_per_task": 40779.12316715543, "last_run_timestamp": "2026-10-24T16:10:43.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.273148, "accuracy_std_dev": 0.006113657170630353, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00006268181818181819, "avg_duration_per_task_ms": 9553.181818181818, "avg_output_tokens_per_task": 586.439393939394, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-4b-it", "model_slug": "google/gemma-3-4b-it", "model_name": "Google: Gemma 3 4B", "provider_name": "Parasail", "accuracy": 0.271044, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011306565656565657, "avg_duration_per_task_ms": 10846.11616161616, "avg_output_tokens_per_task": 1804.4646464646464, "last_run_timestamp": "2026-10-15T03:00:03.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.2700681612244898, "accuracy_std_dev": 0.016752437809465226, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.00005762346938775511, "avg_duration_per_task_ms": 9734.870408163266, "avg_output_tokens_per_task": 576.8010204081633, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.2677151506849315, "accuracy_std_dev": 0.012439954983841373, "run_count": 10, "total_questions": 1971, "avg_cost_per_task_usd": 0.0003856813800101471, "avg_duration_per_task_ms": 392968.5474378488, "avg_output_tokens_per_task": 9238.72602739726, "last_run_timestamp": "2026-10-24T16:01:29.844Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.26430966666666666, "accuracy_std_dev": 0.1964870751683739, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014085552188552189, "avg_duration_per_task_ms": 155316.38383838383, "avg_output_tokens_per_task": 16743.055555555555, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.2575758888888889, "accuracy_std_dev": 0.014825798696449854, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00042795959595959596, "avg_duration_per_task_ms": 75249.49046015713, "avg_output_tokens_per_task": 8235.553310886644, "last_run_timestamp": "2026-10-24T16:02:28.577Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.19163883333333334, "accuracy_std_dev": 0.1344497289267339, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0010863872053872054, "avg_duration_per_task_ms": 449223.712962963, "avg_output_tokens_per_task": 3508.121212121212, "last_run_timestamp": "2026-10-24T16:03:19.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.1861954, "accuracy_std_dev": 0.12390642146491036, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.001524880808080808, "avg_duration_per_task_ms": 49007.062626262625, "avg_output_tokens_per_task": 1318.2202020202021, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.17340049999999999, "accuracy_std_dev": 0.11279449999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00031717929292929297, "avg_duration_per_task_ms": 9334.631313131313, "avg_output_tokens_per_task": 867.929292929293, "last_run_timestamp": "2026-10-13T18:56:05.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.0025255000000000004, "accuracy_std_dev": 0.0008415000000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034195530303030306, "avg_duration_per_task_ms": 159765.32575757575, "avg_output_tokens_per_task": 3219.0782828282827, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034544065656565656, "avg_duration_per_task_ms": 164367.952020202, "avg_output_tokens_per_task": 3253.931818181818, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8368423578947368, "accuracy_std_dev": 0.014073500000000017, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.008221831578947368, "avg_duration_per_task_ms": 31565.694736842106, "avg_output_tokens_per_task": 9295.157894736842, "last_run_timestamp": "2026-10-10T11:02:59.339Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.809028, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.06453804166666667, "avg_duration_per_task_ms": 453773, "avg_output_tokens_per_task": 8614.875, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8013334, "accuracy_std_dev": 0.021249926537284785, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.8758712919999999, "avg_duration_per_task_ms": 139145.43, "avg_output_tokens_per_task": 5724.488, "last_run_timestamp": "2026-10-24T11:41:42.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8011110833333334, "accuracy_std_dev": 0.022580570421855795, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.8756809116666666, "avg_duration_per_task_ms": 131059.94833333333, "avg_output_tokens_per_task": 5774.293333333333, "last_run_timestamp": "2026-10-24T11:46:02.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.8, "accuracy_std_dev": 0.013332999999999984, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015587, "avg_duration_per_task_ms": 194697.61, "avg_output_tokens_per_task": 11724.04, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.7996450638297874, "accuracy_std_dev": 0.011700000000000044, "run_count": 2, "total_questions": 94, "avg_cost_per_task_usd": 0.06506034042553191, "avg_duration_per_task_ms": 304476.60638297873, "avg_output_tokens_per_task": 17465.68085106383, "last_run_timestamp": "2026-10-24T19:25:11.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.7994444166666667, "accuracy_std_dev": 0.026311211407745082, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.9154123233333333, "avg_duration_per_task_ms": 123961.04, "avg_output_tokens_per_task": 5773.983333333334, "last_run_timestamp": "2026-10-24T11:40:21.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.7983334999999999, "accuracy_std_dev": 0.03444005258779377, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04458143, "avg_duration_per_task_ms": 93369.73, "avg_output_tokens_per_task": 5128.475, "last_run_timestamp": "2026-10-24T12:30:27.997Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7944445, "accuracy_std_dev": 0.01651785042764341, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.8238463366666665, "avg_duration_per_task_ms": 131103.42666666667, "avg_output_tokens_per_task": 5759.988333333334, "last_run_timestamp": "2026-10-24T12:06:18.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7940580869565217, "accuracy_std_dev": 0.02103502135862134, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.9401787339130434, "avg_duration_per_task_ms": 94460.84695652174, "avg_output_tokens_per_task": 5691.410434782609, "last_run_timestamp": "2026-10-24T11:52:13.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7936708987341772, "accuracy_std_dev": 0.03037925132800345, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.12511070886075948, "avg_duration_per_task_ms": 2933394.1088607595, "avg_output_tokens_per_task": 27206.47088607595, "last_run_timestamp": "2026-10-17T19:41:14.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00828808, "avg_duration_per_task_ms": 196026.56, "avg_output_tokens_per_task": 16303, "last_run_timestamp": "2026-10-14T19:21:35.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03146312, "avg_duration_per_task_ms": 636210.72, "avg_output_tokens_per_task": 16745.44, "last_run_timestamp": "2026-10-14T19:17:59.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.7920001999999999, "accuracy_std_dev": 0.014847359662916514, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05966906, "avg_duration_per_task_ms": 918022.212, "avg_output_tokens_per_task": 41767.624, "last_run_timestamp": "2026-10-24T12:47:38.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7916668169642858, "accuracy_std_dev": 0.03139197384682907, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.1966235267857143, "avg_duration_per_task_ms": 790382.3013392857, "avg_output_tokens_per_task": 33130.86830357143, "last_run_timestamp": "2026-10-24T19:22:19.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7911647630522088, "accuracy_std_dev": 0.020527481807567145, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.028665044176706823, "avg_duration_per_task_ms": 359430.156626506, "avg_output_tokens_per_task": 12014.867469879518, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.7883330000000001, "accuracy_std_dev": 0.02598076211353313, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06002512, "avg_duration_per_task_ms": 888300.44, "avg_output_tokens_per_task": 41523.41, "last_run_timestamp": "2026-10-24T12:33:32.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.78777775, "accuracy_std_dev": 0.020875937181058474, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.48297491666666664, "avg_duration_per_task_ms": 122364.275, "avg_output_tokens_per_task": 7096.316666666667, "last_run_timestamp": "2026-10-24T11:55:24.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7876542314814815, "accuracy_std_dev": 0.022797342804780508, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.07938424444444445, "avg_duration_per_task_ms": 1140818.088888889, "avg_output_tokens_per_task": 26072.82222222222, "last_run_timestamp": "2026-10-24T19:36:21.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.787288561983471, "accuracy_std_dev": 0.015937097625777535, "run_count": 17, "total_questions": 847, "avg_cost_per_task_usd": 0.09573880283353008, "avg_duration_per_task_ms": 142932.90082644628, "avg_output_tokens_per_task": 11874.43919716647, "last_run_timestamp": "2026-10-24T13:24:07.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.786667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.029090889999999998, "avg_duration_per_task_ms": 149301.66, "avg_output_tokens_per_task": 12176.77, "last_run_timestamp": "2026-10-24T11:38:31.986Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.49828548, "avg_duration_per_task_ms": 348417.68, "avg_output_tokens_per_task": 21971.04, "last_run_timestamp": "2026-10-14T15:16:05.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.78666675, "accuracy_std_dev": 0.024944159861328273, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15998305999999998, "avg_duration_per_task_ms": 73714.275, "avg_output_tokens_per_task": 6299.305, "last_run_timestamp": "2026-10-24T11:26:51.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7862320000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.029582717391304348, "avg_duration_per_task_ms": 2435445.304347826, "avg_output_tokens_per_task": 108721.58695652174, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7860963903743317, "accuracy_std_dev": 0.025611788150185825, "run_count": 4, "total_questions": 187, "avg_cost_per_task_usd": 0.0982251550802139, "avg_duration_per_task_ms": 69951.20320855615, "avg_output_tokens_per_task": 7732.208556149732, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7860360641891891, "accuracy_std_dev": 0.025554613476174253, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.09440423986486486, "avg_duration_per_task_ms": 381879.5878378378, "avg_output_tokens_per_task": 16484.91891891892, "last_run_timestamp": "2026-10-10T11:07:21.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.78583325, "accuracy_std_dev": 0.016811747713652493, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07747623, "avg_duration_per_task_ms": 47946.0625, "avg_output_tokens_per_task": 4735.27, "last_run_timestamp": "2026-10-24T11:58:13.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.7853333, "accuracy_std_dev": 0.011469724399914758, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.17582804400000004, "avg_duration_per_task_ms": 67798.288, "avg_output_tokens_per_task": 6240.81, "last_run_timestamp": "2026-10-24T12:30:32.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7842432787356322, "accuracy_std_dev": 0.019450509073790118, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.09254919540229886, "avg_duration_per_task_ms": 69468.46551724138, "avg_output_tokens_per_task": 4697.962643678161, "last_run_timestamp": "2026-10-24T13:01:18.254Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7840001999999999, "accuracy_std_dev": 0.014966487007978884, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.174428044, "avg_duration_per_task_ms": 71541.076, "avg_output_tokens_per_task": 6317.768, "last_run_timestamp": "2026-10-24T11:35:39.965Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7837036666666667, "accuracy_std_dev": 0.021513329521743285, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18632613555555552, "avg_duration_per_task_ms": 421396.91333333333, "avg_output_tokens_per_task": 30066.795555555556, "last_run_timestamp": "2026-10-24T18:46:07.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.783333375, "accuracy_std_dev": 0.023804813940343558, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.19070967249999998, "avg_duration_per_task_ms": 778974.4025, "avg_output_tokens_per_task": 30803.295, "last_run_timestamp": "2026-10-24T19:20:47.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.7833331666666667, "accuracy_std_dev": 0.018358760147103848, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06827201333333334, "avg_duration_per_task_ms": 62356.823333333334, "avg_output_tokens_per_task": 12364.343333333334, "last_run_timestamp": "2026-10-24T11:44:08.040Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.7831111333333334, "accuracy_std_dev": 0.01851801018780244, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.181601536, "avg_duration_per_task_ms": 65069.36666666667, "avg_output_tokens_per_task": 6347.210666666667, "last_run_timestamp": "2026-10-24T12:24:09.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7831052054794521, "accuracy_std_dev": 0.017155537965592597, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.14992966438356164, "avg_duration_per_task_ms": 183129.3904109589, "avg_output_tokens_per_task": 16313.164383561643, "last_run_timestamp": "2026-10-15T21:26:28.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.782963111111111, "accuracy_std_dev": 0.025936403174964765, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07150440444444443, "avg_duration_per_task_ms": 328376.76444444444, "avg_output_tokens_per_task": 17192.448888888888, "last_run_timestamp": "2026-10-24T19:51:50.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7826668, "accuracy_std_dev": 0.01717857063204037, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.01173768, "avg_duration_per_task_ms": 599702.892, "avg_output_tokens_per_task": 22568, "last_run_timestamp": "2026-10-24T14:27:48.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.78166675, "accuracy_std_dev": 0.015183827290492351, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.014457230000000001, "avg_duration_per_task_ms": 401220.695, "avg_output_tokens_per_task": 21060.795, "last_run_timestamp": "2026-10-24T13:36:02.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7809524285714285, "accuracy_std_dev": 0.025053077443455044, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08778324, "avg_duration_per_task_ms": 269653.90285714285, "avg_output_tokens_per_task": 20106.945714285714, "last_run_timestamp": "2026-10-24T12:40:07.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7809524285714285, "accuracy_std_dev": 0.011507741653303821, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08243106857142857, "avg_duration_per_task_ms": 42328.47714285714, "avg_output_tokens_per_task": 4763.78, "last_run_timestamp": "2026-10-24T12:26:24.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7807406666666666, "accuracy_std_dev": 0.01923075098204262, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08175265555555557, "avg_duration_per_task_ms": 137863.5, "avg_output_tokens_per_task": 4670.331111111111, "last_run_timestamp": "2026-10-24T13:21:36.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7804090721205597, "accuracy_std_dev": 0.030493720143653286, "run_count": 19, "total_questions": 929, "avg_cost_per_task_usd": 0.07802858557588804, "avg_duration_per_task_ms": 76989.98815931109, "avg_output_tokens_per_task": 4254.709364908504, "last_run_timestamp": "2026-10-24T12:32:51.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7800003333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2757919799999997, "avg_duration_per_task_ms": 102078.39333333333, "avg_output_tokens_per_task": 5685.653333333334, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7800001111111111, "accuracy_std_dev": 0.0235177999322709, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06914864222222221, "avg_duration_per_task_ms": 267980.70666666667, "avg_output_tokens_per_task": 14178.462222222222, "last_run_timestamp": "2026-10-24T12:19:13.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.78, "accuracy_std_dev": 0.005443582705045132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12296124000000001, "avg_duration_per_task_ms": 145422.42, "avg_output_tokens_per_task": 4596.633333333333, "last_run_timestamp": "2026-10-24T13:02:59.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7799998, "accuracy_std_dev": 0.01633001326882497, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.055028052, "avg_duration_per_task_ms": 146838.836, "avg_output_tokens_per_task": 4580.112, "last_run_timestamp": "2026-10-17T10:59:26.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7794594324324324, "accuracy_std_dev": 0.024130720890140577, "run_count": 37, "total_questions": 1850, "avg_cost_per_task_usd": 0.5428561410810812, "avg_duration_per_task_ms": 112001.87405405406, "avg_output_tokens_per_task": 7201.864864864865, "last_run_timestamp": "2026-10-24T11:58:58.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7794445, "accuracy_std_dev": 0.022926342067223316, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.47765851166666656, "avg_duration_per_task_ms": 106942.50333333333, "avg_output_tokens_per_task": 7221.848333333333, "last_run_timestamp": "2026-10-24T11:37:51.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7793333, "accuracy_std_dev": 0.02393506864853326, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.09867859, "avg_duration_per_task_ms": 301600.814, "avg_output_tokens_per_task": 20737.49, "last_run_timestamp": "2026-10-24T11:44:08.040Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.7792640401337794, "accuracy_std_dev": 0.028913483122066055, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.1886598294314381, "avg_duration_per_task_ms": 56557.29765886287, "avg_output_tokens_per_task": 4653.71237458194, "last_run_timestamp": "2026-10-24T12:16:13.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7790740277777776, "accuracy_std_dev": 0.020193614874352558, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.4918987627777778, "avg_duration_per_task_ms": 122390.91666666667, "avg_output_tokens_per_task": 7175.807222222223, "last_run_timestamp": "2026-10-24T11:54:38.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7788095, "accuracy_std_dev": 0.023891421292858595, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.13300575857142857, "avg_duration_per_task_ms": 319690.7614285714, "avg_output_tokens_per_task": 19596.084285714285, "last_run_timestamp": "2026-10-24T13:53:01.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.77833325, "accuracy_std_dev": 0.011902275693643617, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0336993, "avg_duration_per_task_ms": 220470.06, "avg_output_tokens_per_task": 12832.385, "last_run_timestamp": "2026-10-24T11:48:43.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7781817272727273, "accuracy_std_dev": 0.03929598310743943, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1061598909090909, "avg_duration_per_task_ms": 180685.46181818182, "avg_output_tokens_per_task": 6783.716363636364, "last_run_timestamp": "2026-10-24T11:43:03.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.777436, "accuracy_std_dev": 0.01913318832652994, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.446252443076923, "avg_duration_per_task_ms": 126382.57538461538, "avg_output_tokens_per_task": 7217.507692307692, "last_run_timestamp": "2026-10-24T11:43:26.286Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7773333333333333, "accuracy_std_dev": 0.022016936243618362, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.49882911999999996, "avg_duration_per_task_ms": 126443.04666666666, "avg_output_tokens_per_task": 8076.794666666667, "last_run_timestamp": "2026-10-24T12:26:01.097Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7771429999999999, "accuracy_std_dev": 0.017036713725699246, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.15415246857142859, "avg_duration_per_task_ms": 315516.36, "avg_output_tokens_per_task": 19966.505714285715, "last_run_timestamp": "2026-10-24T14:03:17.639Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.7771428571428571, "accuracy_std_dev": 0.018120213341763423, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09363316857142856, "avg_duration_per_task_ms": 130500.70857142858, "avg_output_tokens_per_task": 5167.157142857143, "last_run_timestamp": "2026-10-24T13:07:35.631Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7770371111111111, "accuracy_std_dev": 0.019903844298942444, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0686149911111111, "avg_duration_per_task_ms": 43266.05111111111, "avg_output_tokens_per_task": 4744.831111111111, "last_run_timestamp": "2026-10-24T12:49:42.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00755077, "avg_duration_per_task_ms": 173908.24, "avg_output_tokens_per_task": 19171.69, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7766553164983165, "accuracy_std_dev": 0.020416356422812885, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.0789071414141414, "avg_duration_per_task_ms": 233953.09764309763, "avg_output_tokens_per_task": 11808.565656565657, "last_run_timestamp": "2026-10-24T19:28:54.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7764444, "accuracy_std_dev": 0.017017152714442756, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.43634274400000006, "avg_duration_per_task_ms": 93543.88666666667, "avg_output_tokens_per_task": 5699.424, "last_run_timestamp": "2026-10-24T12:19:07.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7759999000000001, "accuracy_std_dev": 0.02174607193242035, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08178571999999999, "avg_duration_per_task_ms": 336667.87, "avg_output_tokens_per_task": 20194.232, "last_run_timestamp": "2026-10-24T12:55:53.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7758335, "accuracy_std_dev": 0.014883874571159226, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12466751749999999, "avg_duration_per_task_ms": 254510.0675, "avg_output_tokens_per_task": 23726.7775, "last_run_timestamp": "2026-10-24T12:19:24.121Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7755444170854271, "accuracy_std_dev": 0.02087028870637634, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.15095997738693467, "avg_duration_per_task_ms": 341068.3417085427, "avg_output_tokens_per_task": 20195.442211055277, "last_run_timestamp": "2026-10-24T13:15:20.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7753625384615385, "accuracy_std_dev": 0.017323958686038136, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06710363879598663, "avg_duration_per_task_ms": 1125092.9698996656, "avg_output_tokens_per_task": 36573.67558528428, "last_run_timestamp": "2026-10-24T13:26:35.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7753333, "accuracy_std_dev": 0.025307041621058755, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.48832906800000003, "avg_duration_per_task_ms": 125426.254, "avg_output_tokens_per_task": 7994.034, "last_run_timestamp": "2026-10-24T12:05:00.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7746666, "accuracy_std_dev": 0.02508632562652409, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.16812989199999998, "avg_duration_per_task_ms": 82939.64, "avg_output_tokens_per_task": 6304.564, "last_run_timestamp": "2026-10-24T11:50:50.507Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7745533125, "accuracy_std_dev": 0.016606913050345267, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.08764765178571429, "avg_duration_per_task_ms": 134355.6294642857, "avg_output_tokens_per_task": 4417.004464285715, "last_run_timestamp": "2026-10-24T12:55:13.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7743589230769231, "accuracy_std_dev": 0.016086511681250408, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.12578370615384615, "avg_duration_per_task_ms": 93781.64923076922, "avg_output_tokens_per_task": 4504.9, "last_run_timestamp": "2026-10-24T13:03:36.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.026945723560596418, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21623159142857146, "avg_duration_per_task_ms": 562306.5571428571, "avg_output_tokens_per_task": 12602.945714285714, "last_run_timestamp": "2026-10-10T11:06:43.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7740741111111111, "accuracy_std_dev": 0.022098307604602994, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05655496222222222, "avg_duration_per_task_ms": 155063.69555555555, "avg_output_tokens_per_task": 16071.293333333333, "last_run_timestamp": "2026-10-24T11:33:51.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7739130434782608, "accuracy_std_dev": 0.029285348993887062, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.5129370408695653, "avg_duration_per_task_ms": 128405.80086956522, "avg_output_tokens_per_task": 7970.952173913043, "last_run_timestamp": "2026-10-24T11:35:03.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7738888333333334, "accuracy_std_dev": 0.018749437853054562, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4345813583333333, "avg_duration_per_task_ms": 98445.79833333334, "avg_output_tokens_per_task": 5637.391666666666, "last_run_timestamp": "2026-10-24T11:36:40.498Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7738887916666667, "accuracy_std_dev": 0.031145842469981935, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.4443571025000001, "avg_duration_per_task_ms": 79517.7575, "avg_output_tokens_per_task": 5656.515, "last_run_timestamp": "2026-10-24T12:17:57.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7738695201005025, "accuracy_std_dev": 0.034154093534853214, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.18507628140703516, "avg_duration_per_task_ms": 590522.5326633166, "avg_output_tokens_per_task": 28427.434673366835, "last_run_timestamp": "2026-10-24T19:18:27.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.7733335, "accuracy_std_dev": 0.02708037419885479, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13855224, "avg_duration_per_task_ms": 310185.72, "avg_output_tokens_per_task": 21233.035, "last_run_timestamp": "2026-10-24T12:39:23.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01881542, "avg_duration_per_task_ms": 96553.18, "avg_output_tokens_per_task": 11186.19, "last_run_timestamp": "2026-10-24T11:34:09.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.7733335, "accuracy_std_dev": 0.03126950946609175, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.068859715, "avg_duration_per_task_ms": 704699.005, "avg_output_tokens_per_task": 13178.825, "last_run_timestamp": "2026-10-24T12:26:26.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.7733334, "accuracy_std_dev": 0.013984054413509715, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.17623406799999997, "avg_duration_per_task_ms": 74817.192, "avg_output_tokens_per_task": 6393.728, "last_run_timestamp": "2026-10-24T11:36:43.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.7733332857142857, "accuracy_std_dev": 0.022537305934283946, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08668051142857143, "avg_duration_per_task_ms": 364495.4428571428, "avg_output_tokens_per_task": 20483.94, "last_run_timestamp": "2026-10-24T12:45:56.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0051899, "avg_duration_per_task_ms": 123477.22, "avg_output_tokens_per_task": 11624.46, "last_run_timestamp": "2026-10-20T20:17:11.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11485508, "avg_duration_per_task_ms": 381375.48, "avg_output_tokens_per_task": 14199.1, "last_run_timestamp": "2026-10-13T03:50:19.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.77277775, "accuracy_std_dev": 0.02530680323919834, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.47471332, "avg_duration_per_task_ms": 112995.27166666667, "avg_output_tokens_per_task": 7220.64, "last_run_timestamp": "2026-10-24T11:37:27.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7726668, "accuracy_std_dev": 0.0325508757356849, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.065897038, "avg_duration_per_task_ms": 245761.102, "avg_output_tokens_per_task": 11799.874, "last_run_timestamp": "2026-10-24T12:58:13.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7726668, "accuracy_std_dev": 0.027030792318391252, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.058849759999999994, "avg_duration_per_task_ms": 890114.912, "avg_output_tokens_per_task": 40932.588, "last_run_timestamp": "2026-10-24T13:04:00.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.7725925555555556, "accuracy_std_dev": 0.027115371049200147, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02128270888888889, "avg_duration_per_task_ms": 231570.07111111112, "avg_output_tokens_per_task": 12386.437777777777, "last_run_timestamp": "2026-10-24T19:11:05.620Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.7723808571428572, "accuracy_std_dev": 0.03553283429420726, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0410159, "avg_duration_per_task_ms": 732886.1685714286, "avg_output_tokens_per_task": 12629.145714285714, "last_run_timestamp": "2026-10-24T12:41:08.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.7722223333333333, "accuracy_std_dev": 0.051949695516164705, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2672586, "avg_duration_per_task_ms": 26529.913333333334, "avg_output_tokens_per_task": 12210.046666666667, "last_run_timestamp": "2026-10-24T12:34:49.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7722222222222223, "accuracy_std_dev": 0.03095202181433487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03593436666666667, "avg_duration_per_task_ms": 446735.1666666667, "avg_output_tokens_per_task": 13815.786666666667, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.771929874493927, "accuracy_std_dev": 0.030837182396580943, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.011004716599190284, "avg_duration_per_task_ms": 1072772.7165991904, "avg_output_tokens_per_task": 18798.943319838058, "last_run_timestamp": "2026-10-17T12:03:02.755Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.771867598108747, "accuracy_std_dev": 0.030418188929615536, "run_count": 17, "total_questions": 846, "avg_cost_per_task_usd": 0.03435285106382978, "avg_duration_per_task_ms": 501754.59338061465, "avg_output_tokens_per_task": 12458.040189125295, "last_run_timestamp": "2026-10-24T19:21:21.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7718518888888889, "accuracy_std_dev": 0.020313150614233704, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04094091999999999, "avg_duration_per_task_ms": 200471.00222222222, "avg_output_tokens_per_task": 13143.968888888889, "last_run_timestamp": "2026-10-24T19:48:57.247Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7718518888888889, "accuracy_std_dev": 0.02102986445480936, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05397537555555555, "avg_duration_per_task_ms": 110633.31777777777, "avg_output_tokens_per_task": 4708.497777777778, "last_run_timestamp": "2026-10-24T13:04:30.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7714285714285715, "accuracy_std_dev": 0.02435571983191243, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21782835142857143, "avg_duration_per_task_ms": 377656.6857142857, "avg_output_tokens_per_task": 17802.317142857144, "last_run_timestamp": "2026-10-24T12:37:29.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.7714285714285715, "accuracy_std_dev": 0.023289574128825135, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.044844714285714286, "avg_duration_per_task_ms": 100224.04857142858, "avg_output_tokens_per_task": 4956.551428571429, "last_run_timestamp": "2026-10-24T12:38:46.757Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7713333, "accuracy_std_dev": 0.0351567708046402, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.060528674000000005, "avg_duration_per_task_ms": 302911.6, "avg_output_tokens_per_task": 12614.202, "last_run_timestamp": "2026-10-24T12:18:13.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7711946416184972, "accuracy_std_dev": 0.02510296875371517, "run_count": 14, "total_questions": 692, "avg_cost_per_task_usd": 0.054176160404624286, "avg_duration_per_task_ms": 260129.0578034682, "avg_output_tokens_per_task": 11031.143063583815, "last_run_timestamp": "2026-10-24T12:08:01.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.771111, "accuracy_std_dev": 0.04085521560339636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4149127733333333, "avg_duration_per_task_ms": 472514.2266666667, "avg_output_tokens_per_task": 36470.113333333335, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.77083325, "accuracy_std_dev": 0.040679972209153495, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11823473500000002, "avg_duration_per_task_ms": 271446.195, "avg_output_tokens_per_task": 11914.5925, "last_run_timestamp": "2026-10-17T21:03:47.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.770833, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019129833333333335, "avg_duration_per_task_ms": 643112.25, "avg_output_tokens_per_task": 13416, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7707407777777777, "accuracy_std_dev": 0.023188894510460925, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11877632888888891, "avg_duration_per_task_ms": 486710.77777777775, "avg_output_tokens_per_task": 21345.64, "last_run_timestamp": "2026-10-17T11:37:24.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7706664000000001, "accuracy_std_dev": 0.02686579651229421, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.079545256, "avg_duration_per_task_ms": 210434.584, "avg_output_tokens_per_task": 9339.868, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7704762000000002, "accuracy_std_dev": 0.027384900751848115, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.5769547302857142, "avg_duration_per_task_ms": 116495.80342857142, "avg_output_tokens_per_task": 8134.803428571428, "last_run_timestamp": "2026-10-24T12:27:09.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7704761904761904, "accuracy_std_dev": 0.01947142566164718, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.06930062190476191, "avg_duration_per_task_ms": 74426.91428571429, "avg_output_tokens_per_task": 4672.551428571429, "last_run_timestamp": "2026-10-24T12:23:14.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.770416625, "accuracy_std_dev": 0.0211762125800242, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.079187595, "avg_duration_per_task_ms": 95610.7625, "avg_output_tokens_per_task": 4413.6475, "last_run_timestamp": "2026-10-24T13:02:59.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7703703333333334, "accuracy_std_dev": 0.030325214851304485, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10216244444444443, "avg_duration_per_task_ms": 234870.98, "avg_output_tokens_per_task": 11319.771111111111, "last_run_timestamp": "2026-10-24T19:49:41.079Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7700000833333334, "accuracy_std_dev": 0.020276053200998522, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5360124333333335, "avg_duration_per_task_ms": 174546.98666666666, "avg_output_tokens_per_task": 10626.863333333333, "last_run_timestamp": "2026-10-24T11:55:03.241Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.77, "accuracy_std_dev": 0.018855956915521412, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.065556992, "avg_duration_per_task_ms": 170454.924, "avg_output_tokens_per_task": 10274.532, "last_run_timestamp": "2026-10-24T19:27:30.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.77, "accuracy_std_dev": 0.03454471498507404, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.017919986000000002, "avg_duration_per_task_ms": 291502.854, "avg_output_tokens_per_task": 11512.416, "last_run_timestamp": "2026-10-24T19:22:55.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.77, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10142027, "avg_duration_per_task_ms": 175426.11, "avg_output_tokens_per_task": 10178.13, "last_run_timestamp": "2026-10-24T11:42:19.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.77, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05546486, "avg_duration_per_task_ms": 677432.76, "avg_output_tokens_per_task": 33432.56, "last_run_timestamp": "2026-10-24T13:08:41.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.77, "accuracy_std_dev": 0.022852121225391757, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.047772247500000004, "avg_duration_per_task_ms": 117312.3025, "avg_output_tokens_per_task": 4701.9325, "last_run_timestamp": "2026-10-24T12:40:31.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7697778666666667, "accuracy_std_dev": 0.027741171258298072, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.5468098066666667, "avg_duration_per_task_ms": 184159.76266666668, "avg_output_tokens_per_task": 10508.508, "last_run_timestamp": "2026-10-24T12:18:32.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7696297777777777, "accuracy_std_dev": 0.028477632067516422, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1409867, "avg_duration_per_task_ms": 323727.4288888889, "avg_output_tokens_per_task": 20180.546666666665, "last_run_timestamp": "2026-10-24T12:56:30.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7695238571428571, "accuracy_std_dev": 0.029354336946199627, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12993965428571427, "avg_duration_per_task_ms": 178407.48, "avg_output_tokens_per_task": 8114.634285714285, "last_run_timestamp": "2026-10-17T12:47:59.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.7695237142857143, "accuracy_std_dev": 0.02386662296246901, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11461242285714283, "avg_duration_per_task_ms": 472929.0742857143, "avg_output_tokens_per_task": 11699.722857142857, "last_run_timestamp": "2026-10-23T07:26:58.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7694444166666666, "accuracy_std_dev": 0.029496861625316274, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.16601085666666665, "avg_duration_per_task_ms": 573194.7333333333, "avg_output_tokens_per_task": 34801.78666666667, "last_run_timestamp": "2026-10-24T18:49:47.645Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.769144, "accuracy_std_dev": 0.05227625153611027, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.03692847297297297, "avg_duration_per_task_ms": 119447.77702702703, "avg_output_tokens_per_task": 4304.4121621621625, "last_run_timestamp": "2026-10-24T12:53:05.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7688890833333333, "accuracy_std_dev": 0.019876007624178182, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4929804266666666, "avg_duration_per_task_ms": 127454.61666666667, "avg_output_tokens_per_task": 8112.49, "last_run_timestamp": "2026-10-24T11:39:26.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7688888888888888, "accuracy_std_dev": 0.02914421358022683, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08110119333333334, "avg_duration_per_task_ms": 48308.12, "avg_output_tokens_per_task": 4858.82, "last_run_timestamp": "2026-10-24T12:49:25.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.021998731726069008, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09783254, "avg_duration_per_task_ms": 102292.53333333334, "avg_output_tokens_per_task": 23933.02, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7688235294117647, "accuracy_std_dev": 0.026394292226378947, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.04698485764705882, "avg_duration_per_task_ms": 91736.21058823529, "avg_output_tokens_per_task": 4343.287058823529, "last_run_timestamp": "2026-10-24T12:48:21.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7685714285714286, "accuracy_std_dev": 0.017715488952222757, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.029891637142857145, "avg_duration_per_task_ms": 131231.82571428572, "avg_output_tokens_per_task": 12135.551428571429, "last_run_timestamp": "2026-10-24T18:43:12.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.768333375, "accuracy_std_dev": 0.032787198976649036, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05900729999999999, "avg_duration_per_task_ms": 114247.045, "avg_output_tokens_per_task": 6145.8525, "last_run_timestamp": "2026-10-24T12:32:52.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7681483333333333, "accuracy_std_dev": 0.02753713940602159, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1370029888888889, "avg_duration_per_task_ms": 279741.1088888889, "avg_output_tokens_per_task": 11838.22, "last_run_timestamp": "2026-10-24T19:21:41.094Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.7680176486486487, "accuracy_std_dev": 0.018065309426264085, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06668277702702702, "avg_duration_per_task_ms": 92879.05405405405, "avg_output_tokens_per_task": 4281.108108108108, "last_run_timestamp": "2026-10-24T13:05:47.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.768, "accuracy_std_dev": 0.028875333573830795, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.15511289399999997, "avg_duration_per_task_ms": 603929.03, "avg_output_tokens_per_task": 12401.186, "last_run_timestamp": "2026-10-24T12:59:12.979Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.768, "accuracy_std_dev": 0.011469922022402766, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030947572, "avg_duration_per_task_ms": 58222.324, "avg_output_tokens_per_task": 4803.824, "last_run_timestamp": "2026-10-24T12:19:36.645Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7679999714285717, "accuracy_std_dev": 0.020082359314277674, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.5321429325714288, "avg_duration_per_task_ms": 119418.44171428571, "avg_output_tokens_per_task": 8085.556571428571, "last_run_timestamp": "2026-10-24T12:05:52.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7679998, "accuracy_std_dev": 0.04721585044622197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07315105200000001, "avg_duration_per_task_ms": 1015313.192, "avg_output_tokens_per_task": 46954.436, "last_run_timestamp": "2026-10-24T13:12:06.662Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7679165625000001, "accuracy_std_dev": 0.021885852467886505, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.019775938749999996, "avg_duration_per_task_ms": 242328.7, "avg_output_tokens_per_task": 16787.80875, "last_run_timestamp": "2026-10-24T12:30:46.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7675584849498328, "accuracy_std_dev": 0.01898112793159446, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.01286576254180602, "avg_duration_per_task_ms": 339011.96321070235, "avg_output_tokens_per_task": 16426.635451505015, "last_run_timestamp": "2026-10-24T15:07:38.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.767500125, "accuracy_std_dev": 0.01839017891727471, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05273846000000001, "avg_duration_per_task_ms": 114111.54, "avg_output_tokens_per_task": 4459.3875, "last_run_timestamp": "2026-10-24T13:02:56.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7675, "accuracy_std_dev": 0.019561171859068173, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0578507225, "avg_duration_per_task_ms": 50974.42, "avg_output_tokens_per_task": 4677.1725, "last_run_timestamp": "2026-10-24T12:36:03.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.7673333999999999, "accuracy_std_dev": 0.027235571513004833, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.044897582, "avg_duration_per_task_ms": 196958.324, "avg_output_tokens_per_task": 9659.966, "last_run_timestamp": "2026-10-24T19:32:27.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.767037, "accuracy_std_dev": 0.029247427051128982, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11265956888888888, "avg_duration_per_task_ms": 168759.15333333332, "avg_output_tokens_per_task": 9396.86, "last_run_timestamp": "2026-10-24T20:38:25.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.30783086, "avg_duration_per_task_ms": 74384.36, "avg_output_tokens_per_task": 8511.9, "last_run_timestamp": "2026-10-24T11:55:43.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.76666675, "accuracy_std_dev": 0.01666670000142501, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.15229175250000002, "avg_duration_per_task_ms": 351001.0475, "avg_output_tokens_per_task": 16911.185, "last_run_timestamp": "2026-10-24T20:11:33.520Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.05192598721470999, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02354878666666667, "avg_duration_per_task_ms": 257749.54666666666, "avg_output_tokens_per_task": 13740.606666666667, "last_run_timestamp": "2026-10-24T19:12:17.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.01721334538471048, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1022698822222222, "avg_duration_per_task_ms": 247139.55333333334, "avg_output_tokens_per_task": 17955.155555555557, "last_run_timestamp": "2026-10-24T19:31:34.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.014401440259764175, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3460653, "avg_duration_per_task_ms": 170899.55333333334, "avg_output_tokens_per_task": 12611.153333333334, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7666666, "accuracy_std_dev": 0.03169302061400902, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.435956584, "avg_duration_per_task_ms": 91965.526, "avg_output_tokens_per_task": 5586.83, "last_run_timestamp": "2026-10-24T11:44:03.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.27949224, "avg_duration_per_task_ms": 363191.33, "avg_output_tokens_per_task": 35460.86, "last_run_timestamp": "2026-10-15T21:31:19.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.7666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01352679, "avg_duration_per_task_ms": 90848.6, "avg_output_tokens_per_task": 4507.38, "last_run_timestamp": "2026-10-10T11:18:50.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7666663333333334, "accuracy_std_dev": 0.024944382578492925, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06548607333333334, "avg_duration_per_task_ms": 180351.34, "avg_output_tokens_per_task": 8330.653333333334, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7663690982142857, "accuracy_std_dev": 0.02313623805704157, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.09936721205357145, "avg_duration_per_task_ms": 46610.31919642857, "avg_output_tokens_per_task": 4301.232142857143, "last_run_timestamp": "2026-10-24T12:21:21.947Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7662500624999999, "accuracy_std_dev": 0.022418780493340704, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.01870876375, "avg_duration_per_task_ms": 186867.03125, "avg_output_tokens_per_task": 18628.71375, "last_run_timestamp": "2026-10-24T11:38:35.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7661538461538462, "accuracy_std_dev": 0.023745191285965687, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.09027926615384615, "avg_duration_per_task_ms": 183761.18615384615, "avg_output_tokens_per_task": 11217.321538461538, "last_run_timestamp": "2026-10-24T20:01:36.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7661111666666667, "accuracy_std_dev": 0.02300705876766714, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.41899941, "avg_duration_per_task_ms": 94556.285, "avg_output_tokens_per_task": 5609.2, "last_run_timestamp": "2026-10-24T12:23:35.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7659258888888889, "accuracy_std_dev": 0.023400799584840606, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.049908673333333334, "avg_duration_per_task_ms": 223450.7577777778, "avg_output_tokens_per_task": 12477.913333333334, "last_run_timestamp": "2026-10-24T19:41:33.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.7658335, "accuracy_std_dev": 0.029095183832552104, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.11472753, "avg_duration_per_task_ms": 363943.795, "avg_output_tokens_per_task": 19034.395, "last_run_timestamp": "2026-10-24T12:25:43.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.765833375, "accuracy_std_dev": 0.0314650846333579, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.56580197, "avg_duration_per_task_ms": 182718.72916666666, "avg_output_tokens_per_task": 10626.925, "last_run_timestamp": "2026-10-24T12:21:43.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7657141428571429, "accuracy_std_dev": 0.02265784142050462, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01868964, "avg_duration_per_task_ms": 296461.9485714286, "avg_output_tokens_per_task": 14208.894285714287, "last_run_timestamp": "2026-10-24T13:09:04.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7656249117647059, "accuracy_std_dev": 0.03944849186508596, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.09746370772058824, "avg_duration_per_task_ms": 725535.5073529412, "avg_output_tokens_per_task": 10412.310661764706, "last_run_timestamp": "2026-10-24T19:52:35.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7655824146341464, "accuracy_std_dev": 0.030599644599243303, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.07519595528455283, "avg_duration_per_task_ms": 226711.57317073172, "avg_output_tokens_per_task": 10093.731707317073, "last_run_timestamp": "2026-10-10T19:17:08.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7651851111111111, "accuracy_std_dev": 0.030107298743897986, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009121904444444445, "avg_duration_per_task_ms": 193621.67333333334, "avg_output_tokens_per_task": 4487.197777777777, "last_run_timestamp": "2026-10-24T13:10:16.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.7651515252525253, "accuracy_std_dev": 0.02642279772753065, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.07224725252525253, "avg_duration_per_task_ms": 56760.18939393939, "avg_output_tokens_per_task": 4516.429292929293, "last_run_timestamp": "2026-10-24T12:38:39.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7650000277777779, "accuracy_std_dev": 0.02201993695556385, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.6106352950000001, "avg_duration_per_task_ms": 72113.83055555556, "avg_output_tokens_per_task": 5697.888888888889, "last_run_timestamp": "2026-10-24T12:22:02.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.765, "accuracy_std_dev": 0.03840572873934304, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08522504, "avg_duration_per_task_ms": 558480.23, "avg_output_tokens_per_task": 11022.365, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.765, "accuracy_std_dev": 0.02682235800596212, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.093536965, "avg_duration_per_task_ms": 395836.625, "avg_output_tokens_per_task": 14656.925, "last_run_timestamp": "2026-10-24T12:32:30.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.7648484545454546, "accuracy_std_dev": 0.029417433910596247, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.012073312727272727, "avg_duration_per_task_ms": 225241.07636363636, "avg_output_tokens_per_task": 5963.167272727273, "last_run_timestamp": "2026-10-24T13:10:13.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7647713411371238, "accuracy_std_dev": 0.037669588541078686, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.09814772575250837, "avg_duration_per_task_ms": 324868.6220735786, "avg_output_tokens_per_task": 16071.638795986622, "last_run_timestamp": "2026-10-24T19:50:25.118Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7646666, "accuracy_std_dev": 0.021868245879356662, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019245832, "avg_duration_per_task_ms": 612553.372, "avg_output_tokens_per_task": 12922.088, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7645742242152466, "accuracy_std_dev": 0.03220926464716497, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.05124673542600897, "avg_duration_per_task_ms": 396898.4529147982, "avg_output_tokens_per_task": 12964.300448430493, "last_run_timestamp": "2026-10-24T12:42:05.792Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7636566923076923, "accuracy_std_dev": 0.016849090853317095, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028887086956521736, "avg_duration_per_task_ms": 106220.35785953177, "avg_output_tokens_per_task": 18019.585284280936, "last_run_timestamp": "2026-10-24T11:41:59.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7635555333333334, "accuracy_std_dev": 0.026203346503494974, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.5199028973333334, "avg_duration_per_task_ms": 219812.216, "avg_output_tokens_per_task": 10143.609333333334, "last_run_timestamp": "2026-10-24T12:29:32.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.7633335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01887093, "avg_duration_per_task_ms": 108015.59, "avg_output_tokens_per_task": 13223.41, "last_run_timestamp": "2026-10-24T12:18:28.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7633333333333333, "accuracy_std_dev": 0.026317266547944456, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06636648333333332, "avg_duration_per_task_ms": 1144475.0366666666, "avg_output_tokens_per_task": 64035, "last_run_timestamp": "2026-10-24T19:42:45.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7633332777777778, "accuracy_std_dev": 0.01808766376046748, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.010060852222222218, "avg_duration_per_task_ms": 327666.52, "avg_output_tokens_per_task": 14799.732222222223, "last_run_timestamp": "2026-10-24T13:02:06.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.03605539719220273, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.157135485, "avg_duration_per_task_ms": 411704.8, "avg_output_tokens_per_task": 26150.13, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01189864, "avg_duration_per_task_ms": 642869.1, "avg_output_tokens_per_task": 23140.98, "last_run_timestamp": "2026-10-24T13:28:08.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7631776302895322, "accuracy_std_dev": 0.02017755686675451, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.011049703786191535, "avg_duration_per_task_ms": 296930.06904231623, "avg_output_tokens_per_task": 12882.661469933184, "last_run_timestamp": "2026-10-24T19:59:46.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.763025131092437, "accuracy_std_dev": 0.03297689657885083, "run_count": 12, "total_questions": 595, "avg_cost_per_task_usd": 0.19578441512605044, "avg_duration_per_task_ms": 529598.2705882353, "avg_output_tokens_per_task": 27119.653781512607, "last_run_timestamp": "2026-10-24T19:14:04.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.762963, "accuracy_std_dev": 0.025745351040959277, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08904649555555556, "avg_duration_per_task_ms": 300656.5133333333, "avg_output_tokens_per_task": 12058.504444444445, "last_run_timestamp": "2026-10-24T12:17:31.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7629628888888889, "accuracy_std_dev": 0.02354127566374736, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01074868, "avg_duration_per_task_ms": 1059485.7177777777, "avg_output_tokens_per_task": 13128.448888888888, "last_run_timestamp": "2026-10-24T19:49:45.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7629628888888889, "accuracy_std_dev": 0.031128845034949126, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.008403426666666667, "avg_duration_per_task_ms": 205005.57333333333, "avg_output_tokens_per_task": 4237.573333333334, "last_run_timestamp": "2026-10-24T13:15:23.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7629022732240437, "accuracy_std_dev": 0.028478290118661355, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.01388206375227687, "avg_duration_per_task_ms": 226746.868852459, "avg_output_tokens_per_task": 15915.701275045538, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.762857, "accuracy_std_dev": 0.03062451702709541, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03217967714285715, "avg_duration_per_task_ms": 195766.24, "avg_output_tokens_per_task": 14823.671428571428, "last_run_timestamp": "2026-10-24T11:41:00.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7625627587939698, "accuracy_std_dev": 0.017640643984489356, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.01927761809045226, "avg_duration_per_task_ms": 121120.84924623116, "avg_output_tokens_per_task": 14540.939698492462, "last_run_timestamp": "2026-10-24T11:49:30.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.762499875, "accuracy_std_dev": 0.0406800464676403, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0091677075, "avg_duration_per_task_ms": 191141.5625, "avg_output_tokens_per_task": 15674.26, "last_run_timestamp": "2026-10-24T19:03:39.853Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7624696569343066, "accuracy_std_dev": 0.03657436469296787, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.19588743978102188, "avg_duration_per_task_ms": 477892.8375912409, "avg_output_tokens_per_task": 27946.843065693432, "last_run_timestamp": "2026-10-24T19:19:34.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7624473392405063, "accuracy_std_dev": 0.03272282893772328, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.01104873670886076, "avg_duration_per_task_ms": 137884.06329113923, "avg_output_tokens_per_task": 12786.113924050633, "last_run_timestamp": "2026-10-24T19:21:18.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011449053333333332, "avg_duration_per_task_ms": 599408.32, "avg_output_tokens_per_task": 23110.746666666666, "last_run_timestamp": "2026-10-24T14:24:42.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.7622221666666666, "accuracy_std_dev": 0.029855561759448156, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.54613815, "avg_duration_per_task_ms": 190695.96833333332, "avg_output_tokens_per_task": 10651.453333333333, "last_run_timestamp": "2026-10-24T12:29:33.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.762143986599665, "accuracy_std_dev": 0.04074106053036531, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.06808563986599665, "avg_duration_per_task_ms": 207102.986599665, "avg_output_tokens_per_task": 11782.765494137353, "last_run_timestamp": "2026-10-24T12:36:04.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.762, "accuracy_std_dev": 0.02386079071196091, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5457165, "avg_duration_per_task_ms": 180407.204, "avg_output_tokens_per_task": 10745.142, "last_run_timestamp": "2026-10-24T11:51:51.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.7617847878787878, "accuracy_std_dev": 0.03876959831491683, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.02413183333333333, "avg_duration_per_task_ms": 113963.81313131313, "avg_output_tokens_per_task": 15997.81313131313, "last_run_timestamp": "2026-10-24T12:54:33.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.76166675, "accuracy_std_dev": 0.011902275693643652, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.080519975, "avg_duration_per_task_ms": 148683.485, "avg_output_tokens_per_task": 8604.13, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.76166675, "accuracy_std_dev": 0.02229839929653024, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.065100265, "avg_duration_per_task_ms": 116150.4225, "avg_output_tokens_per_task": 4604.0675, "last_run_timestamp": "2026-10-24T12:22:07.097Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7615039124579126, "accuracy_std_dev": 0.01612860952882591, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.1581539494949495, "avg_duration_per_task_ms": 807649.4747474748, "avg_output_tokens_per_task": 30684.649831649833, "last_run_timestamp": "2026-10-24T18:54:54.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7614941091954023, "accuracy_std_dev": 0.008329841446907167, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.007317272988505747, "avg_duration_per_task_ms": 290620.7356321839, "avg_output_tokens_per_task": 12448.798850574713, "last_run_timestamp": "2026-10-24T20:14:29.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.761111, "accuracy_std_dev": 0.016063062451890448, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05401205, "avg_duration_per_task_ms": 73585.80666666667, "avg_output_tokens_per_task": 4477.26, "last_run_timestamp": "2026-10-24T12:38:41.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7610546224489795, "accuracy_std_dev": 0.016475935184003962, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.2764416326530612, "avg_duration_per_task_ms": 159672.02551020408, "avg_output_tokens_per_task": 7623.877551020408, "last_run_timestamp": "2026-10-24T12:38:26.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.760833375, "accuracy_std_dev": 0.017460477198071506, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17206684999999997, "avg_duration_per_task_ms": 195271.6725, "avg_output_tokens_per_task": 11186.875, "last_run_timestamp": "2026-10-24T19:43:42.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.02887947805006949, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18594795111111112, "avg_duration_per_task_ms": 175274.95555555556, "avg_output_tokens_per_task": 9597.25111111111, "last_run_timestamp": "2026-10-24T19:21:02.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.034633054297881376, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.006436806666666667, "avg_duration_per_task_ms": 125547.04444444444, "avg_output_tokens_per_task": 4262.8533333333335, "last_run_timestamp": "2026-10-24T14:25:23.734Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7606666, "accuracy_std_dev": 0.031048465341140474, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.027007422000000003, "avg_duration_per_task_ms": 150781.166, "avg_output_tokens_per_task": 8984.718, "last_run_timestamp": "2026-10-24T19:16:17.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7606059090909091, "accuracy_std_dev": 0.02402931176124294, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03610860181818182, "avg_duration_per_task_ms": 72225.87454545454, "avg_output_tokens_per_task": 4745.927272727273, "last_run_timestamp": "2026-10-24T12:22:54.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.7603748473895582, "accuracy_std_dev": 0.02026019621425223, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.03634712449799197, "avg_duration_per_task_ms": 152002.0281124498, "avg_output_tokens_per_task": 23794.738955823294, "last_run_timestamp": "2026-10-24T12:53:48.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7603120367892977, "accuracy_std_dev": 0.027860534151775026, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06456417725752508, "avg_duration_per_task_ms": 376665.0802675585, "avg_output_tokens_per_task": 52393.28428093645, "last_run_timestamp": "2026-10-24T19:21:23.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019761673333333334, "avg_duration_per_task_ms": 291193.25333333336, "avg_output_tokens_per_task": 17811.393333333333, "last_run_timestamp": "2026-10-10T11:23:23.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.32747625999999996, "avg_duration_per_task_ms": 2629117.52, "avg_output_tokens_per_task": 11787.92, "last_run_timestamp": "2026-10-24T12:30:30.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.02108195647941624, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08341662, "avg_duration_per_task_ms": 533566.115, "avg_output_tokens_per_task": 14923.395, "last_run_timestamp": "2026-10-24T12:36:09.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10151575999999998, "avg_duration_per_task_ms": 434936.03, "avg_output_tokens_per_task": 13962.46, "last_run_timestamp": "2026-10-24T12:32:07.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7595238571428571, "accuracy_std_dev": 0.02174373003832582, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.04527854285714286, "avg_duration_per_task_ms": 136160.14428571428, "avg_output_tokens_per_task": 4500.364285714286, "last_run_timestamp": "2026-10-24T12:17:41.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.7594445, "accuracy_std_dev": 0.03225570769776, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.01983395166666667, "avg_duration_per_task_ms": 192524.21833333332, "avg_output_tokens_per_task": 14719.435, "last_run_timestamp": "2026-10-24T12:16:56.429Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.759408620967742, "accuracy_std_dev": 0.023253272624729635, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.012160826612903226, "avg_duration_per_task_ms": 309190.9919354839, "avg_output_tokens_per_task": 4125.725806451613, "last_run_timestamp": "2026-10-17T11:47:18.976Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7593733097072419, "accuracy_std_dev": 0.029570836436633496, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.08832456086286594, "avg_duration_per_task_ms": 143648.46687211093, "avg_output_tokens_per_task": 12389.725731895223, "last_run_timestamp": "2026-10-24T19:07:01.850Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7593333, "accuracy_std_dev": 0.016452043441773433, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.083344808, "avg_duration_per_task_ms": 106115.21, "avg_output_tokens_per_task": 23741.31, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7592594444444444, "accuracy_std_dev": 0.017340275597650393, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.15033249555555556, "avg_duration_per_task_ms": 165755.92888888888, "avg_output_tokens_per_task": 20125.08, "last_run_timestamp": "2026-10-24T11:39:34.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7592592436412315, "accuracy_std_dev": 0.02502658689065603, "run_count": 15, "total_questions": 747, "avg_cost_per_task_usd": 0.05607525167336011, "avg_duration_per_task_ms": 275516.45113788487, "avg_output_tokens_per_task": 12015.718875502009, "last_run_timestamp": "2026-10-24T11:39:58.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.759166625, "accuracy_std_dev": 0.02777639760631993, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0257473675, "avg_duration_per_task_ms": 253688.2675, "avg_output_tokens_per_task": 13099.1275, "last_run_timestamp": "2026-10-24T19:09:33.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7590477142857143, "accuracy_std_dev": 0.013997181448055748, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.24529245999999996, "avg_duration_per_task_ms": 151053.55142857143, "avg_output_tokens_per_task": 23224.54, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7588886666666668, "accuracy_std_dev": 0.011811189506377245, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.010121153333333334, "avg_duration_per_task_ms": 244060.09, "avg_output_tokens_per_task": 16283.283333333333, "last_run_timestamp": "2026-10-24T13:01:16.778Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.7585186666666667, "accuracy_std_dev": 0.029444037268463494, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.33338275111111115, "avg_duration_per_task_ms": 628430.3377777778, "avg_output_tokens_per_task": 27569.568888888887, "last_run_timestamp": "2026-10-24T18:52:57.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.7585185555555556, "accuracy_std_dev": 0.020793455498792097, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02593214222222222, "avg_duration_per_task_ms": 205207.60444444444, "avg_output_tokens_per_task": 13619.955555555556, "last_run_timestamp": "2026-10-24T20:44:23.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7584615384615384, "accuracy_std_dev": 0.02110672856848985, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.021854490769230767, "avg_duration_per_task_ms": 124647.87076923077, "avg_output_tokens_per_task": 17096.62, "last_run_timestamp": "2026-10-24T12:20:00.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.7583335, "accuracy_std_dev": 0.034760921546328406, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12354291499999999, "avg_duration_per_task_ms": 1021866.15, "avg_output_tokens_per_task": 26151.615, "last_run_timestamp": "2026-10-24T12:47:02.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.40697005500000005, "avg_duration_per_task_ms": 402356.405, "avg_output_tokens_per_task": 35595.7, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.03460084697211181, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0886916525, "avg_duration_per_task_ms": 581833.97, "avg_output_tokens_per_task": 12683.3375, "last_run_timestamp": "2026-10-24T12:19:32.894Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7582857142857146, "accuracy_std_dev": 0.02995624352529768, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.20326376057142856, "avg_duration_per_task_ms": 118129.27428571429, "avg_output_tokens_per_task": 7790.011428571429, "last_run_timestamp": "2026-10-24T12:18:37.751Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7581818181818182, "accuracy_std_dev": 0.034942042529720004, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0916818309090909, "avg_duration_per_task_ms": 244976.56, "avg_output_tokens_per_task": 12409.656363636364, "last_run_timestamp": "2026-10-24T12:33:35.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.758, "accuracy_std_dev": 0.02952224773285396, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.10296491399999999, "avg_duration_per_task_ms": 278724.296, "avg_output_tokens_per_task": 12256.98, "last_run_timestamp": "2026-10-24T12:28:42.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7579964141414142, "accuracy_std_dev": 0.023533015514113357, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20713992929292932, "avg_duration_per_task_ms": 664979.7727272727, "avg_output_tokens_per_task": 53099.67929292929, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.7579907534246576, "accuracy_std_dev": 0.01133630408123485, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.06852380136986301, "avg_duration_per_task_ms": 156936.7876712329, "avg_output_tokens_per_task": 10408.410958904109, "last_run_timestamp": "2026-10-24T18:33:39.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.7579486923076924, "accuracy_std_dev": 0.021623878857143158, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.0068238523076923074, "avg_duration_per_task_ms": 261267.2046153846, "avg_output_tokens_per_task": 15563.432307692308, "last_run_timestamp": "2026-10-24T13:09:56.475Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.757778, "accuracy_std_dev": 0.025724014817546128, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03362461666666666, "avg_duration_per_task_ms": 226709.77666666667, "avg_output_tokens_per_task": 35231.33666666667, "last_run_timestamp": "2026-10-24T11:55:02.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7577777777777778, "accuracy_std_dev": 0.03142696805391397, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04451809777777778, "avg_duration_per_task_ms": 185433.6288888889, "avg_output_tokens_per_task": 9376.346666666666, "last_run_timestamp": "2026-10-24T19:48:57.855Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7577777777777778, "accuracy_std_dev": 0.022662374322886335, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06675582666666667, "avg_duration_per_task_ms": 168314.9311111111, "avg_output_tokens_per_task": 4664.111111111111, "last_run_timestamp": "2026-10-24T14:02:36.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.04629468145358481, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12420428666666666, "avg_duration_per_task_ms": 116279.86, "avg_output_tokens_per_task": 19029.046666666665, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.35916456, "avg_duration_per_task_ms": 165675.65333333332, "avg_output_tokens_per_task": 12758.866666666667, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.7575758811188811, "accuracy_std_dev": 0.010732768960317529, "run_count": 3, "total_questions": 143, "avg_cost_per_task_usd": 0.27724311888111886, "avg_duration_per_task_ms": 338699.5594405594, "avg_output_tokens_per_task": 6540.4965034965035, "last_run_timestamp": "2026-10-24T11:53:36.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7575504530201342, "accuracy_std_dev": 0.028065656837905205, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.19407245637583895, "avg_duration_per_task_ms": 482897.4211409396, "avg_output_tokens_per_task": 20396.08389261745, "last_run_timestamp": "2026-10-24T14:01:08.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7575376909547739, "accuracy_std_dev": 0.025888825973329584, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.019801728643216082, "avg_duration_per_task_ms": 423933.55778894474, "avg_output_tokens_per_task": 16756.603015075376, "last_run_timestamp": "2026-10-24T12:17:05.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7574406116071428, "accuracy_std_dev": 0.06727408355010252, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.25612264285714287, "avg_duration_per_task_ms": 471727.34375, "avg_output_tokens_per_task": 29191.647321428572, "last_run_timestamp": "2026-10-24T19:02:15.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7574072222222222, "accuracy_std_dev": 0.030298005023351983, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015441244444444443, "avg_duration_per_task_ms": 72124.47777777778, "avg_output_tokens_per_task": 4310.673333333333, "last_run_timestamp": "2026-10-17T11:14:15.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.7573529926470588, "accuracy_std_dev": 0.018618180288085897, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.03312612867647059, "avg_duration_per_task_ms": 167023.56985294117, "avg_output_tokens_per_task": 16331.753676470587, "last_run_timestamp": "2026-10-24T12:47:41.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.032000033334982636, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.028732296, "avg_duration_per_task_ms": 171140.192, "avg_output_tokens_per_task": 15777.572, "last_run_timestamp": "2026-10-24T12:23:36.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7573332666666667, "accuracy_std_dev": 0.01944421688991929, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.02663897066666666, "avg_duration_per_task_ms": 283461.33733333333, "avg_output_tokens_per_task": 16864.825333333334, "last_run_timestamp": "2026-10-24T13:28:03.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.75722225, "accuracy_std_dev": 0.026311355692061323, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5355160966666667, "avg_duration_per_task_ms": 190831.025, "avg_output_tokens_per_task": 10368.328333333333, "last_run_timestamp": "2026-10-24T11:54:04.367Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7571428571428571, "accuracy_std_dev": 0.024651550802730272, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.012830634285714286, "avg_duration_per_task_ms": 221282.58285714284, "avg_output_tokens_per_task": 4997.4742857142855, "last_run_timestamp": "2026-10-24T12:58:11.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.757037111111111, "accuracy_std_dev": 0.02311772460181651, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014514133333333333, "avg_duration_per_task_ms": 244582.29555555555, "avg_output_tokens_per_task": 15046.255555555555, "last_run_timestamp": "2026-10-24T19:19:42.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.757037, "accuracy_std_dev": 0.01666255916718679, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.17754851555555554, "avg_duration_per_task_ms": 320852.2622222222, "avg_output_tokens_per_task": 10861.113333333333, "last_run_timestamp": "2026-10-24T11:53:33.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7568475720930232, "accuracy_std_dev": 0.02459803490467476, "run_count": 13, "total_questions": 645, "avg_cost_per_task_usd": 0.09061050697674419, "avg_duration_per_task_ms": 702536.334883721, "avg_output_tokens_per_task": 21508.96434108527, "last_run_timestamp": "2026-10-24T12:57:12.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023368849999999997, "avg_duration_per_task_ms": 302412.09, "avg_output_tokens_per_task": 11606.21, "last_run_timestamp": "2026-10-24T12:08:01.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.03853813887854761, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.19762862999999997, "avg_duration_per_task_ms": 102033.36166666666, "avg_output_tokens_per_task": 7661.711666666667, "last_run_timestamp": "2026-10-24T12:18:01.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.04895203245581898, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02191566, "avg_duration_per_task_ms": 332956.19333333336, "avg_output_tokens_per_task": 12348.723333333333, "last_run_timestamp": "2026-10-24T12:55:32.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.7566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018845109999999998, "avg_duration_per_task_ms": 92258.91, "avg_output_tokens_per_task": 11470.03, "last_run_timestamp": "2026-10-24T12:23:52.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.7566665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.010595590000000002, "avg_duration_per_task_ms": 149870.75, "avg_output_tokens_per_task": 13511.14, "last_run_timestamp": "2026-10-24T12:17:44.341Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.7564445999999999, "accuracy_std_dev": 0.020921957750331767, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.25126205733333334, "avg_duration_per_task_ms": 263807.58666666667, "avg_output_tokens_per_task": 10194.865333333333, "last_run_timestamp": "2026-10-24T12:05:03.668Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7563351081871345, "accuracy_std_dev": 0.013338594901982123, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.033182684210526316, "avg_duration_per_task_ms": 2146298.769005848, "avg_output_tokens_per_task": 50213.23391812865, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7562962222222223, "accuracy_std_dev": 0.08255839119176704, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07683437111111112, "avg_duration_per_task_ms": 192163.8777777778, "avg_output_tokens_per_task": 23858.006666666668, "last_run_timestamp": "2026-10-24T18:45:10.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7562961111111112, "accuracy_std_dev": 0.02650155778836174, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013197128888888889, "avg_duration_per_task_ms": 222309.89333333334, "avg_output_tokens_per_task": 14339.34888888889, "last_run_timestamp": "2026-10-24T19:54:36.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7561111666666667, "accuracy_std_dev": 0.02924456159366313, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.20050754833333334, "avg_duration_per_task_ms": 123808.41166666667, "avg_output_tokens_per_task": 7847.445, "last_run_timestamp": "2026-10-24T11:58:02.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.756, "accuracy_std_dev": 0.024073867807230344, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07011482399999999, "avg_duration_per_task_ms": 334056.692, "avg_output_tokens_per_task": 14633.816, "last_run_timestamp": "2026-10-24T12:59:11.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7559648947368421, "accuracy_std_dev": 0.020191449941144954, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.29131978947368414, "avg_duration_per_task_ms": 264712.04, "avg_output_tokens_per_task": 11238.59157894737, "last_run_timestamp": "2026-10-24T12:24:12.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7559523303571429, "accuracy_std_dev": 0.02997543982811411, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.02883498660714286, "avg_duration_per_task_ms": 493727.48883928574, "avg_output_tokens_per_task": 12075.370535714286, "last_run_timestamp": "2026-10-24T19:23:19.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.755892303030303, "accuracy_std_dev": 0.022789500000000018, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.027549767676767676, "avg_duration_per_task_ms": 104945.30303030302, "avg_output_tokens_per_task": 10081.373737373737, "last_run_timestamp": "2026-10-24T12:19:36.298Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.75583325, "accuracy_std_dev": 0.019982613972839007, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.008674565, "avg_duration_per_task_ms": 119249.875, "avg_output_tokens_per_task": 4714.72, "last_run_timestamp": "2026-10-24T12:46:25.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7556522173913043, "accuracy_std_dev": 0.03227948562565843, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.20032831565217393, "avg_duration_per_task_ms": 100773.6104347826, "avg_output_tokens_per_task": 7631.826086956522, "last_run_timestamp": "2026-10-24T12:48:21.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7556052242152467, "accuracy_std_dev": 0.025010430481878735, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.020433513452914796, "avg_duration_per_task_ms": 233457.43946188342, "avg_output_tokens_per_task": 4244.6569506726455, "last_run_timestamp": "2026-10-24T12:41:47.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7555556, "accuracy_std_dev": 0.02330687812585232, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.028939747999999994, "avg_duration_per_task_ms": 174269.36933333334, "avg_output_tokens_per_task": 17673.329333333335, "last_run_timestamp": "2026-10-24T12:19:39.208Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7555555555555555, "accuracy_std_dev": 0.02330683151215116, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.007330602222222223, "avg_duration_per_task_ms": 193346.8088888889, "avg_output_tokens_per_task": 4415.6, "last_run_timestamp": "2026-10-24T14:44:52.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7554201788617886, "accuracy_std_dev": 0.023120046817426663, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.025219461382113822, "avg_duration_per_task_ms": 380343.8069105691, "avg_output_tokens_per_task": 12305.247967479674, "last_run_timestamp": "2026-10-24T19:18:15.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7553846153846154, "accuracy_std_dev": 0.022400361015423448, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.023595866153846155, "avg_duration_per_task_ms": 114714.57846153846, "avg_output_tokens_per_task": 14825.998461538462, "last_run_timestamp": "2026-10-24T12:22:58.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7553335, "accuracy_std_dev": 0.031134064945811373, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.085864622, "avg_duration_per_task_ms": 71299.624, "avg_output_tokens_per_task": 12122.088, "last_run_timestamp": "2026-10-24T19:37:32.789Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.755319, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.014918489361702128, "avg_duration_per_task_ms": 876881.4042553192, "avg_output_tokens_per_task": 9452.702127659575, "last_run_timestamp": "2026-10-24T14:20:18.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7551851666666668, "accuracy_std_dev": 0.03354053891064762, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.22096761222222225, "avg_duration_per_task_ms": 95197.85388888889, "avg_output_tokens_per_task": 7539.6577777777775, "last_run_timestamp": "2026-10-24T12:46:28.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.755, "accuracy_std_dev": 0.04014562657874455, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.161091306, "avg_duration_per_task_ms": 311751.64, "avg_output_tokens_per_task": 23196.802, "last_run_timestamp": "2026-10-24T12:52:46.316Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7548484545454546, "accuracy_std_dev": 0.021805424301655014, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.01682924181818182, "avg_duration_per_task_ms": 119778.08181818182, "avg_output_tokens_per_task": 4470.658181818182, "last_run_timestamp": "2026-10-24T13:29:07.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7548147777777778, "accuracy_std_dev": 0.027716071710502704, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021845168888888888, "avg_duration_per_task_ms": 476468.12444444443, "avg_output_tokens_per_task": 14231.342222222222, "last_run_timestamp": "2026-10-24T18:47:38.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7545454545454545, "accuracy_std_dev": 0.03163137195128692, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.008343416363636365, "avg_duration_per_task_ms": 139678.11272727273, "avg_output_tokens_per_task": 4440.807272727273, "last_run_timestamp": "2026-10-24T13:04:30.804Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7542857142857143, "accuracy_std_dev": 0.014876559353697425, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05917716571428572, "avg_duration_per_task_ms": 372934.26285714284, "avg_output_tokens_per_task": 14097.657142857142, "last_run_timestamp": "2026-10-24T13:33:32.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7541274103773585, "accuracy_std_dev": 0.021560862869117733, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.007241307783018868, "avg_duration_per_task_ms": 204331.1474056604, "avg_output_tokens_per_task": 13534.349056603774, "last_run_timestamp": "2026-10-24T12:31:25.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7539998, "accuracy_std_dev": 0.010832125062055002, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09518565200000001, "avg_duration_per_task_ms": 207718.192, "avg_output_tokens_per_task": 9678.708, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7537880151515151, "accuracy_std_dev": 0.02397695833731574, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.01387228787878788, "avg_duration_per_task_ms": 200605.87121212122, "avg_output_tokens_per_task": 4857.164141414141, "last_run_timestamp": "2026-10-17T11:28:07.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7537201651785714, "accuracy_std_dev": 0.044602290458818805, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.11035437723214285, "avg_duration_per_task_ms": 289472.76116071426, "avg_output_tokens_per_task": 11809.0625, "last_run_timestamp": "2026-10-24T19:31:21.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7534912732240436, "accuracy_std_dev": 0.02055401911633615, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.027412590163934426, "avg_duration_per_task_ms": 556770.8961748633, "avg_output_tokens_per_task": 22246.586520947178, "last_run_timestamp": "2026-10-24T20:27:36.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.7534013163265306, "accuracy_std_dev": 0.008336108804472242, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.03488917857142857, "avg_duration_per_task_ms": 592188.2653061225, "avg_output_tokens_per_task": 12444.612244897959, "last_run_timestamp": "2026-10-24T12:44:02.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.753333375, "accuracy_std_dev": 0.01452975873799614, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1921822125, "avg_duration_per_task_ms": 294611.6675, "avg_output_tokens_per_task": 11751.955, "last_run_timestamp": "2026-10-24T11:55:23.806Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11451421333333334, "avg_duration_per_task_ms": 146387.56666666668, "avg_output_tokens_per_task": 17546.113333333335, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.29158895333333334, "avg_duration_per_task_ms": 171961.5, "avg_output_tokens_per_task": 10696.2, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.012765636746968635, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17238889666666665, "avg_duration_per_task_ms": 505286.37666666665, "avg_output_tokens_per_task": 9803.186666666666, "last_run_timestamp": "2026-10-03T16:16:10.656Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7533331428571429, "accuracy_std_dev": 0.0230940520074423, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.011956362857142858, "avg_duration_per_task_ms": 437970.4885714286, "avg_output_tokens_per_task": 6008.582857142857, "last_run_timestamp": "2026-10-24T12:34:15.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.024162740000000002, "avg_duration_per_task_ms": 65962.82, "avg_output_tokens_per_task": 8135.38, "last_run_timestamp": "2026-10-24T12:37:26.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01197713, "avg_duration_per_task_ms": 212560.45, "avg_output_tokens_per_task": 13957.42, "last_run_timestamp": "2026-10-24T11:49:49.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0185477, "avg_duration_per_task_ms": 97588.58, "avg_output_tokens_per_task": 10717.4, "last_run_timestamp": "2026-10-24T11:41:40.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.032659863237109024, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12209446666666665, "avg_duration_per_task_ms": 147223.81333333332, "avg_output_tokens_per_task": 19831.306666666667, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.7533329999999999, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0194427, "avg_duration_per_task_ms": 127110.83, "avg_output_tokens_per_task": 12815.84, "last_run_timestamp": "2026-10-24T12:03:02.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7527778333333334, "accuracy_std_dev": 0.026311368008376572, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.501247295, "avg_duration_per_task_ms": 196372.42666666667, "avg_output_tokens_per_task": 10155.583333333334, "last_run_timestamp": "2026-10-24T12:42:05.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7527272727272727, "accuracy_std_dev": 0.029875735292863623, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.013207607272727274, "avg_duration_per_task_ms": 136477.7381818182, "avg_output_tokens_per_task": 16262.556363636364, "last_run_timestamp": "2026-10-24T11:51:53.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.03021638651133634, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.5157013133333334, "avg_duration_per_task_ms": 178548.1511111111, "avg_output_tokens_per_task": 7818.084444444445, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7525337229729728, "accuracy_std_dev": 0.02409323915008227, "run_count": 12, "total_questions": 592, "avg_cost_per_task_usd": 0.014006842905405404, "avg_duration_per_task_ms": 57600.26858108108, "avg_output_tokens_per_task": 4072.2314189189187, "last_run_timestamp": "2026-10-24T12:20:04.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7524445999999999, "accuracy_std_dev": 0.02858304219824523, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.35143002133333334, "avg_duration_per_task_ms": 230047.736, "avg_output_tokens_per_task": 10847.656, "last_run_timestamp": "2026-10-24T11:48:43.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.7522223333333334, "accuracy_std_dev": 0.033866116211077726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.024190810000000004, "avg_duration_per_task_ms": 128051.35, "avg_output_tokens_per_task": 10623.603333333333, "last_run_timestamp": "2026-10-24T18:55:24.450Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.7522223333333333, "accuracy_std_dev": 0.03297965328636568, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.014283683333333335, "avg_duration_per_task_ms": 164802.21666666667, "avg_output_tokens_per_task": 14931.153333333334, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.75222225, "accuracy_std_dev": 0.04696988331211997, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.13807676666666668, "avg_duration_per_task_ms": 829053.1233333333, "avg_output_tokens_per_task": 31392.836666666666, "last_run_timestamp": "2026-10-24T19:43:40.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7522221666666666, "accuracy_std_dev": 0.01511278994115765, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.44058496666666663, "avg_duration_per_task_ms": 458389.25, "avg_output_tokens_per_task": 35681.92333333333, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7521905714285715, "accuracy_std_dev": 0.023501964796022273, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.37098750228571425, "avg_duration_per_task_ms": 159626.864, "avg_output_tokens_per_task": 11846.640571428572, "last_run_timestamp": "2026-10-24T12:39:24.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.752, "accuracy_std_dev": 0.01857135189478677, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.33952648, "avg_duration_per_task_ms": 120001.212, "avg_output_tokens_per_task": 7107.392, "last_run_timestamp": "2026-10-24T12:22:52.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7518518333333334, "accuracy_std_dev": 0.017715982107345284, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.36871217222222225, "avg_duration_per_task_ms": 149259.27444444445, "avg_output_tokens_per_task": 7150.745555555555, "last_run_timestamp": "2026-10-24T12:24:28.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7516666666666667, "accuracy_std_dev": 0.021963791754208158, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.34539720999999995, "avg_duration_per_task_ms": 219756.98166666666, "avg_output_tokens_per_task": 11912.11, "last_run_timestamp": "2026-10-24T12:39:07.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7514285714285714, "accuracy_std_dev": 0.021296004758486267, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.009337331428571431, "avg_duration_per_task_ms": 230829.7457142857, "avg_output_tokens_per_task": 4265.0371428571425, "last_run_timestamp": "2026-10-24T13:34:36.032Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7513888333333334, "accuracy_std_dev": 0.027452241920692434, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.5585198816666668, "avg_duration_per_task_ms": 202142.5925, "avg_output_tokens_per_task": 9870.238333333333, "last_run_timestamp": "2026-10-24T12:58:30.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7511113333333332, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48754586, "avg_duration_per_task_ms": 141651.29333333333, "avg_output_tokens_per_task": 7913.286666666667, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7511111666666667, "accuracy_std_dev": 0.02454511593248011, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3494615833333333, "avg_duration_per_task_ms": 100555.84, "avg_output_tokens_per_task": 7111.15, "last_run_timestamp": "2026-10-24T11:44:08.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.751111, "accuracy_std_dev": 0.01523486904002351, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3475124233333333, "avg_duration_per_task_ms": 124013.22666666667, "avg_output_tokens_per_task": 7125.08, "last_run_timestamp": "2026-10-24T11:31:04.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.751111, "accuracy_std_dev": 0.018921497998308716, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03688384, "avg_duration_per_task_ms": 282057.35833333334, "avg_output_tokens_per_task": 21913.646666666667, "last_run_timestamp": "2026-10-24T19:02:49.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.751111, "accuracy_std_dev": 0.03861798340151904, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4379569333333333, "avg_duration_per_task_ms": 39099.41333333333, "avg_output_tokens_per_task": 9339.68, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.7506668, "accuracy_std_dev": 0.025854254755455656, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.264866352, "avg_duration_per_task_ms": 264537.464, "avg_output_tokens_per_task": 10440.212, "last_run_timestamp": "2026-10-24T11:49:28.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.7506666, "accuracy_std_dev": 0.024073823498563778, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022084748, "avg_duration_per_task_ms": 137222.84, "avg_output_tokens_per_task": 13584.26, "last_run_timestamp": "2026-10-24T19:40:37.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7505555, "accuracy_std_dev": 0.03574174773170259, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03120006, "avg_duration_per_task_ms": 258878.755, "avg_output_tokens_per_task": 17037.54, "last_run_timestamp": "2026-10-24T12:34:23.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.7503703333333334, "accuracy_std_dev": 0.021798491033402586, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09827831777777778, "avg_duration_per_task_ms": 290553.8022222222, "avg_output_tokens_per_task": 12820.055555555555, "last_run_timestamp": "2026-10-24T19:44:46.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7503347710843373, "accuracy_std_dev": 0.02780906636854249, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.004999224899598394, "avg_duration_per_task_ms": 263913.3353413655, "avg_output_tokens_per_task": 12415.361445783132, "last_run_timestamp": "2026-10-24T20:11:39.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7502788193979932, "accuracy_std_dev": 0.03287689055865162, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.01826405016722408, "avg_duration_per_task_ms": 135784.91304347827, "avg_output_tokens_per_task": 4013.7859531772574, "last_run_timestamp": "2026-10-24T12:16:33.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7500001666666666, "accuracy_std_dev": 0.023333261905247805, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.3534387216666667, "avg_duration_per_task_ms": 87888.735, "avg_output_tokens_per_task": 7005.235, "last_run_timestamp": "2026-10-24T12:23:35.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7500001, "accuracy_std_dev": 0.02333341428702625, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.196243942, "avg_duration_per_task_ms": 117146.822, "avg_output_tokens_per_task": 7664.984, "last_run_timestamp": "2026-10-24T12:32:11.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02619736, "avg_duration_per_task_ms": 832580.72, "avg_output_tokens_per_task": 18187.39, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.75, "accuracy_std_dev": 0.010000000000000009, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.300415095, "avg_duration_per_task_ms": 278099.495, "avg_output_tokens_per_task": 10648.38, "last_run_timestamp": "2026-10-24T12:46:10.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.75, "accuracy_std_dev": 0.04384306609373936, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.50077502, "avg_duration_per_task_ms": 139265.845, "avg_output_tokens_per_task": 7986.47, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.75, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09412079000000001, "avg_duration_per_task_ms": 469968.05, "avg_output_tokens_per_task": 4703.42, "last_run_timestamp": "2026-10-24T12:22:38.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7496296666666666, "accuracy_std_dev": 0.022246872749020503, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06072559777777777, "avg_duration_per_task_ms": 205446.72222222222, "avg_output_tokens_per_task": 10600.893333333333, "last_run_timestamp": "2026-10-24T20:08:17.474Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7494445, "accuracy_std_dev": 0.024412988600674583, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08046244666666666, "avg_duration_per_task_ms": 794170.4033333333, "avg_output_tokens_per_task": 35486.02166666667, "last_run_timestamp": "2026-10-24T19:16:16.979Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7491623969849246, "accuracy_std_dev": 0.02175219497401357, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.11730947487437185, "avg_duration_per_task_ms": 117420.94974874372, "avg_output_tokens_per_task": 10306.74623115578, "last_run_timestamp": "2026-10-17T19:14:08.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7488889166666667, "accuracy_std_dev": 0.027397460908687426, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02489698, "avg_duration_per_task_ms": 157509.05666666667, "avg_output_tokens_per_task": 8743.431666666667, "last_run_timestamp": "2026-10-24T19:12:17.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.7488888888888889, "accuracy_std_dev": 0.02330689507632377, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.007562466666666667, "avg_duration_per_task_ms": 152556.50444444444, "avg_output_tokens_per_task": 12996.975555555555, "last_run_timestamp": "2026-10-24T19:02:53.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7488888888888889, "accuracy_std_dev": 0.016329946740366617, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01262068, "avg_duration_per_task_ms": 336899.4511111111, "avg_output_tokens_per_task": 16364.997777777779, "last_run_timestamp": "2026-10-24T12:24:10.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7488888666666667, "accuracy_std_dev": 0.02560864244707678, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.34009223066666666, "avg_duration_per_task_ms": 209111.22666666665, "avg_output_tokens_per_task": 11827.101333333334, "last_run_timestamp": "2026-10-24T13:09:08.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7488737702702704, "accuracy_std_dev": 0.035681240826082414, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.3075426081081082, "avg_duration_per_task_ms": 149494.7972972973, "avg_output_tokens_per_task": 20582.41554054054, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.748479288321168, "accuracy_std_dev": 0.026337582738596148, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.04487602919708029, "avg_duration_per_task_ms": 472772.2098540146, "avg_output_tokens_per_task": 23140.4598540146, "last_run_timestamp": "2026-10-24T19:38:23.863Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7483334166666666, "accuracy_std_dev": 0.03210214167429313, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5444264083333333, "avg_duration_per_task_ms": 182768.61333333334, "avg_output_tokens_per_task": 10413.385, "last_run_timestamp": "2026-10-24T12:39:07.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.748333375, "accuracy_std_dev": 0.028431152223122702, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021531867499999996, "avg_duration_per_task_ms": 145411.285, "avg_output_tokens_per_task": 16569.425, "last_run_timestamp": "2026-10-24T11:58:02.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.748275724137931, "accuracy_std_dev": 0.011196981765934397, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.07340827586206897, "avg_duration_per_task_ms": 233348.6, "avg_output_tokens_per_task": 7130.5931034482755, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.748275551724138, "accuracy_std_dev": 0.011091913731282908, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.00999303448275862, "avg_duration_per_task_ms": 128162.52413793103, "avg_output_tokens_per_task": 3554.88275862069, "last_run_timestamp": "2026-10-24T12:44:22.556Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.747906015075377, "accuracy_std_dev": 0.009429275581931, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.0214910351758794, "avg_duration_per_task_ms": 74311.23618090453, "avg_output_tokens_per_task": 12217.904522613066, "last_run_timestamp": "2026-10-24T13:10:59.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.022580213671978926, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.018034246666666667, "avg_duration_per_task_ms": 75225.08333333333, "avg_output_tokens_per_task": 4268.986666666667, "last_run_timestamp": "2026-10-24T12:38:26.301Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.74777775, "accuracy_std_dev": 0.02650409355843297, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.52115986, "avg_duration_per_task_ms": 227711.20333333334, "avg_output_tokens_per_task": 10054.68, "last_run_timestamp": "2026-10-24T12:47:43.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.747691088161209, "accuracy_std_dev": 0.03008101142513954, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.09758945340050378, "avg_duration_per_task_ms": 397184.00755667506, "avg_output_tokens_per_task": 25302.57934508816, "last_run_timestamp": "2026-10-24T13:02:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7475554666666667, "accuracy_std_dev": 0.02281875035978576, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013525546666666666, "avg_duration_per_task_ms": 139030.82666666666, "avg_output_tokens_per_task": 15549.14, "last_run_timestamp": "2026-10-24T12:22:53.795Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7473276647230321, "accuracy_std_dev": 0.024479710940170173, "run_count": 7, "total_questions": 343, "avg_cost_per_task_usd": 0.13171293294460643, "avg_duration_per_task_ms": 209506.55976676385, "avg_output_tokens_per_task": 11363.093294460641, "last_run_timestamp": "2026-10-24T19:49:56.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7472728181818181, "accuracy_std_dev": 0.03871218656330843, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.023786492727272724, "avg_duration_per_task_ms": 413071.56363636366, "avg_output_tokens_per_task": 15329.412727272727, "last_run_timestamp": "2026-10-17T11:15:55.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7472727272727273, "accuracy_std_dev": 0.0336894846720913, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10508834363636363, "avg_duration_per_task_ms": 310050.1672727273, "avg_output_tokens_per_task": 18603.976363636364, "last_run_timestamp": "2026-10-24T12:22:55.595Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7472726363636364, "accuracy_std_dev": 0.03984348764688863, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.04194175272727273, "avg_duration_per_task_ms": 190838.81636363638, "avg_output_tokens_per_task": 19126.147272727274, "last_run_timestamp": "2026-10-24T12:19:21.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7472221666666666, "accuracy_std_dev": 0.011613686586906374, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17439429333333334, "avg_duration_per_task_ms": 161420.59666666668, "avg_output_tokens_per_task": 12499.9, "last_run_timestamp": "2026-10-10T11:19:48.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7471783002257337, "accuracy_std_dev": 0.040296469031969095, "run_count": 9, "total_questions": 443, "avg_cost_per_task_usd": 0.11695101128668173, "avg_duration_per_task_ms": 466244.8329571106, "avg_output_tokens_per_task": 21735.31828442438, "last_run_timestamp": "2026-10-03T08:18:33.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.747143, "accuracy_std_dev": 0.033305956785965055, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014612914285714287, "avg_duration_per_task_ms": 50786.11142857143, "avg_output_tokens_per_task": 4096.908571428571, "last_run_timestamp": "2026-10-24T13:27:44.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.746766898623279, "accuracy_std_dev": 0.01597020524668026, "run_count": 16, "total_questions": 799, "avg_cost_per_task_usd": 0.045530197747183976, "avg_duration_per_task_ms": 308691.72590738424, "avg_output_tokens_per_task": 26256.58448060075, "last_run_timestamp": "2026-10-24T19:41:04.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.746667, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00955731, "avg_duration_per_task_ms": 207347.51, "avg_output_tokens_per_task": 13127.37, "last_run_timestamp": "2026-10-24T12:22:57.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08652918, "avg_duration_per_task_ms": 60257.98, "avg_output_tokens_per_task": 8644.22, "last_run_timestamp": "2026-10-24T12:40:48.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00930136, "avg_duration_per_task_ms": 89811.58, "avg_output_tokens_per_task": 12020.36, "last_run_timestamp": "2026-10-24T15:18:33.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0524172, "avg_duration_per_task_ms": 255934.74, "avg_output_tokens_per_task": 8431.46, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7466667, "accuracy_std_dev": 0.02875178835150955, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.069247084, "avg_duration_per_task_ms": 254648.06, "avg_output_tokens_per_task": 9736.448, "last_run_timestamp": "2026-10-24T19:33:48.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.023726809342082992, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21412848666666662, "avg_duration_per_task_ms": 142371.99333333335, "avg_output_tokens_per_task": 22756.126666666667, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18399340666666666, "avg_duration_per_task_ms": 116870.40666666666, "avg_output_tokens_per_task": 5117.2266666666665, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.746666625, "accuracy_std_dev": 0.025165982331996805, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0084872125, "avg_duration_per_task_ms": 222581.33, "avg_output_tokens_per_task": 5098.45, "last_run_timestamp": "2026-10-17T11:55:03.480Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.7466666000000001, "accuracy_std_dev": 0.030983694638309366, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022173396, "avg_duration_per_task_ms": 122667.268, "avg_output_tokens_per_task": 7404.18, "last_run_timestamp": "2026-10-24T12:20:04.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.029814069333871523, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02163002857142857, "avg_duration_per_task_ms": 164450.89714285714, "avg_output_tokens_per_task": 8529.557142857142, "last_run_timestamp": "2026-10-24T18:59:48.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7466665555555556, "accuracy_std_dev": 0.03471233459324771, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07409725333333333, "avg_duration_per_task_ms": 215903.96666666667, "avg_output_tokens_per_task": 9487.915555555555, "last_run_timestamp": "2026-10-24T19:32:20.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.7466665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.019887960000000003, "avg_duration_per_task_ms": 51072.42, "avg_output_tokens_per_task": 13340.71, "last_run_timestamp": "2026-10-24T12:26:21.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.7466665, "accuracy_std_dev": 0.0466665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.011416649999999999, "avg_duration_per_task_ms": 95996.46, "avg_output_tokens_per_task": 13669.03, "last_run_timestamp": "2026-10-24T12:07:39.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7459258888888889, "accuracy_std_dev": 0.025809336792221565, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.17245649777777777, "avg_duration_per_task_ms": 324708.56444444443, "avg_output_tokens_per_task": 11055.957777777778, "last_run_timestamp": "2026-10-24T12:33:16.196Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7459258888888889, "accuracy_std_dev": 0.032423682687006024, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10307451777777778, "avg_duration_per_task_ms": 332680.06888888887, "avg_output_tokens_per_task": 7684.475555555556, "last_run_timestamp": "2026-10-24T19:37:38.705Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.74583325, "accuracy_std_dev": 0.03402396010442498, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02499106, "avg_duration_per_task_ms": 929627.78, "avg_output_tokens_per_task": 52531.49, "last_run_timestamp": "2026-10-24T12:31:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.7455556666666666, "accuracy_std_dev": 0.022166719578884227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03517121333333333, "avg_duration_per_task_ms": 100449.04, "avg_output_tokens_per_task": 17842.213333333333, "last_run_timestamp": "2026-10-24T12:20:51.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7455556666666666, "accuracy_std_dev": 0.020846357411627908, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.018734011111111108, "avg_duration_per_task_ms": 376517.3044444444, "avg_output_tokens_per_task": 24556.922222222223, "last_run_timestamp": "2026-10-24T11:55:52.478Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7455555833333333, "accuracy_std_dev": 0.027329632768170454, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.3244978516666666, "avg_duration_per_task_ms": 197778.63333333333, "avg_output_tokens_per_task": 11507.296666666667, "last_run_timestamp": "2026-10-24T12:37:04.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7455555833333333, "accuracy_std_dev": 0.10487495876237436, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.15016765666666665, "avg_duration_per_task_ms": 591828.39, "avg_output_tokens_per_task": 27540.295, "last_run_timestamp": "2026-10-24T19:26:44.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7454957162162162, "accuracy_std_dev": 0.015757653442057918, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.12553497972972974, "avg_duration_per_task_ms": 598677.7297297297, "avg_output_tokens_per_task": 24585.641891891893, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7453334, "accuracy_std_dev": 0.02543827737170895, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.346131274, "avg_duration_per_task_ms": 200655.086, "avg_output_tokens_per_task": 11839.45, "last_run_timestamp": "2026-10-24T13:00:12.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.02368052570633507, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03867792222222222, "avg_duration_per_task_ms": 612636.5933333334, "avg_output_tokens_per_task": 15988.397777777778, "last_run_timestamp": "2026-10-24T19:37:04.226Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.03170500430103102, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.13362894444444445, "avg_duration_per_task_ms": 336443.09555555554, "avg_output_tokens_per_task": 11874.128888888888, "last_run_timestamp": "2026-10-24T19:38:37.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.04034135688727106, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11528159555555555, "avg_duration_per_task_ms": 255335.0888888889, "avg_output_tokens_per_task": 9618.166666666666, "last_run_timestamp": "2026-10-24T19:48:41.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7451851111111111, "accuracy_std_dev": 0.030270966502370805, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4099032177777778, "avg_duration_per_task_ms": 619397.1977777778, "avg_output_tokens_per_task": 9848.733333333334, "last_run_timestamp": "2026-10-24T12:05:22.704Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.745000125, "accuracy_std_dev": 0.03312433437609538, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.015775572499999998, "avg_duration_per_task_ms": 148938.23, "avg_output_tokens_per_task": 15126.975, "last_run_timestamp": "2026-10-24T12:05:54.269Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.745, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15284815, "avg_duration_per_task_ms": 494527.49, "avg_output_tokens_per_task": 28246.935, "last_run_timestamp": "2026-10-17T19:47:21.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.745, "accuracy_std_dev": 0.028332999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01250761, "avg_duration_per_task_ms": 355978.98, "avg_output_tokens_per_task": 23604.2, "last_run_timestamp": "2026-10-24T12:29:47.596Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.745, "accuracy_std_dev": 0.040380248197602775, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007515549999999999, "avg_duration_per_task_ms": 328112.605, "avg_output_tokens_per_task": 3998.705, "last_run_timestamp": "2026-10-24T13:13:17.945Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7448485454545455, "accuracy_std_dev": 0.021944100032763564, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.4515615263636363, "avg_duration_per_task_ms": 587567.8727272727, "avg_output_tokens_per_task": 10197.047272727274, "last_run_timestamp": "2026-10-24T11:44:25.226Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7446668, "accuracy_std_dev": 0.025130744341543096, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.079246066, "avg_duration_per_task_ms": 290947.32, "avg_output_tokens_per_task": 11272.336, "last_run_timestamp": "2026-10-24T23:01:57.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7445560603015074, "accuracy_std_dev": 0.018800290223823686, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.0472956608040201, "avg_duration_per_task_ms": 744919.2989949749, "avg_output_tokens_per_task": 14283.391959798995, "last_run_timestamp": "2026-10-24T19:19:10.662Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.00831485358994506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.2319787333333332, "avg_duration_per_task_ms": 98531.56, "avg_output_tokens_per_task": 5524.42, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7444070973154362, "accuracy_std_dev": 0.02348016052421823, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.1433310234899329, "avg_duration_per_task_ms": 614878.5134228187, "avg_output_tokens_per_task": 26835.629194630874, "last_run_timestamp": "2026-10-24T20:02:49.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7442960934891485, "accuracy_std_dev": 0.02614118042622386, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.011909569282136896, "avg_duration_per_task_ms": 437548.9916527546, "avg_output_tokens_per_task": 14218.791318864774, "last_run_timestamp": "2026-10-24T12:42:25.209Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7442423636363636, "accuracy_std_dev": 0.04102458399827359, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.011285496363636365, "avg_duration_per_task_ms": 301862.60363636364, "avg_output_tokens_per_task": 16012.072727272727, "last_run_timestamp": "2026-10-24T12:46:15.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7442423636363636, "accuracy_std_dev": 0.026858821460209414, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02083015818181818, "avg_duration_per_task_ms": 406483.71090909094, "avg_output_tokens_per_task": 12085.40909090909, "last_run_timestamp": "2026-10-17T11:29:07.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.74416675, "accuracy_std_dev": 0.03012731088543251, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.064155875, "avg_duration_per_task_ms": 1722921.32, "avg_output_tokens_per_task": 8915.945, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.744131569416499, "accuracy_std_dev": 0.0294471423722235, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.06635609456740442, "avg_duration_per_task_ms": 1016068.6076458753, "avg_output_tokens_per_task": 74326.28571428571, "last_run_timestamp": "2026-10-24T19:38:23.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.744036743119266, "accuracy_std_dev": 0.02171918727532931, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.025592662385321107, "avg_duration_per_task_ms": 271579.7009174312, "avg_output_tokens_per_task": 19610.990825688074, "last_run_timestamp": "2026-10-24T12:32:51.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7439515846774194, "accuracy_std_dev": 0.044830576271223645, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.07215105040322582, "avg_duration_per_task_ms": 329377.40524193546, "avg_output_tokens_per_task": 12649.491935483871, "last_run_timestamp": "2026-10-24T19:51:25.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7439125188916877, "accuracy_std_dev": 0.0319877187270267, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.06879249370277078, "avg_duration_per_task_ms": 463688.63979848864, "avg_output_tokens_per_task": 10615.753148614609, "last_run_timestamp": "2026-10-24T19:42:30.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7438111658031088, "accuracy_std_dev": 0.030675189182609607, "run_count": 12, "total_questions": 579, "avg_cost_per_task_usd": 0.021574207253886007, "avg_duration_per_task_ms": 94130.40587219344, "avg_output_tokens_per_task": 7255.53195164076, "last_run_timestamp": "2026-10-24T18:57:18.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7438096428571428, "accuracy_std_dev": 0.027105232069017705, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.06181787428571429, "avg_duration_per_task_ms": 140380.01, "avg_output_tokens_per_task": 9906.655714285715, "last_run_timestamp": "2026-10-24T19:18:50.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7436412530120482, "accuracy_std_dev": 0.036080178615965866, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.07742782530120482, "avg_duration_per_task_ms": 824019.6144578314, "avg_output_tokens_per_task": 14546.923694779116, "last_run_timestamp": "2026-10-24T20:35:30.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06141422999999999, "avg_duration_per_task_ms": 1060280.1, "avg_output_tokens_per_task": 12094.68, "last_run_timestamp": "2026-10-24T12:42:07.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.7433335, "accuracy_std_dev": 0.023333285714832386, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.006871515, "avg_duration_per_task_ms": 97438.585, "avg_output_tokens_per_task": 9380.645, "last_run_timestamp": "2026-10-24T12:26:07.751Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.7433335, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03822515, "avg_duration_per_task_ms": 176603.12, "avg_output_tokens_per_task": 16434.51, "last_run_timestamp": "2026-10-24T18:56:11.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.7433335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03870923, "avg_duration_per_task_ms": 112805.69, "avg_output_tokens_per_task": 14592.13, "last_run_timestamp": "2026-10-24T12:30:47.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.01774311029726281, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.013128406666666667, "avg_duration_per_task_ms": 72447.74833333334, "avg_output_tokens_per_task": 10872.788333333334, "last_run_timestamp": "2026-10-24T12:05:47.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.022110882189263745, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.010621789999999999, "avg_duration_per_task_ms": 218070.5775, "avg_output_tokens_per_task": 14077.9775, "last_run_timestamp": "2026-10-24T19:37:39.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7431372352941178, "accuracy_std_dev": 0.03152894888087569, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.37377429588235295, "avg_duration_per_task_ms": 200222.53705882354, "avg_output_tokens_per_task": 11930.03411764706, "last_run_timestamp": "2026-10-24T13:14:09.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7430303636363637, "accuracy_std_dev": 0.0482721820820095, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009174501818181817, "avg_duration_per_task_ms": 294400.0709090909, "avg_output_tokens_per_task": 15379.543636363636, "last_run_timestamp": "2026-10-24T11:53:54.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7429629444444444, "accuracy_std_dev": 0.018622075637599298, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.35968263222222224, "avg_duration_per_task_ms": 87204.23444444445, "avg_output_tokens_per_task": 7123.5233333333335, "last_run_timestamp": "2026-10-24T12:18:54.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.742962888888889, "accuracy_std_dev": 0.03570948009343192, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.35507732222222216, "avg_duration_per_task_ms": 386491.71777777775, "avg_output_tokens_per_task": 9636.25111111111, "last_run_timestamp": "2026-10-24T12:16:41.815Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7429378258859785, "accuracy_std_dev": 0.02605274813906652, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.051288061633281964, "avg_duration_per_task_ms": 216591.9815100154, "avg_output_tokens_per_task": 9230.906009244993, "last_run_timestamp": "2026-10-24T19:33:32.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.02908997383704564, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.504919826, "avg_duration_per_task_ms": 191444.46, "avg_output_tokens_per_task": 9912.906, "last_run_timestamp": "2026-10-24T11:43:06.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.01768882635564045, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.056281936000000005, "avg_duration_per_task_ms": 224029.02, "avg_output_tokens_per_task": 9140.692, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7425926666666667, "accuracy_std_dev": 0.03165311687654157, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013586744444444443, "avg_duration_per_task_ms": 236886.98, "avg_output_tokens_per_task": 22693.746666666666, "last_run_timestamp": "2026-10-24T12:29:34.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7425595803571429, "accuracy_std_dev": 0.02504151285854947, "run_count": 7, "total_questions": 336, "avg_cost_per_task_usd": 0.15002946726190475, "avg_duration_per_task_ms": 711525.6666666666, "avg_output_tokens_per_task": 29928.91369047619, "last_run_timestamp": "2026-10-24T19:30:34.911Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.7425, "accuracy_std_dev": 0.01697637247028941, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.22007109249999998, "avg_duration_per_task_ms": 261814.77, "avg_output_tokens_per_task": 11695.5675, "last_run_timestamp": "2026-10-24T11:58:56.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7424244545454545, "accuracy_std_dev": 0.04131889777277492, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009867483636363635, "avg_duration_per_task_ms": 412569.9109090909, "avg_output_tokens_per_task": 14715.354545454546, "last_run_timestamp": "2026-10-17T11:26:35.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.006285707880227596, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.38184122000000004, "avg_duration_per_task_ms": 149071.25333333333, "avg_output_tokens_per_task": 12794.173333333334, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0028365266666666666, "avg_duration_per_task_ms": 68565.14666666667, "avg_output_tokens_per_task": 11675.026666666667, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.028284218870442763, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0722646911111111, "avg_duration_per_task_ms": 320607.4111111111, "avg_output_tokens_per_task": 11515.595555555556, "last_run_timestamp": "2026-10-24T22:27:21.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7421122716049383, "accuracy_std_dev": 0.04949233809146621, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.07618280658436215, "avg_duration_per_task_ms": 605963.6255144033, "avg_output_tokens_per_task": 7686.909465020576, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.7417778, "accuracy_std_dev": 0.0192398742345162, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.016580628, "avg_duration_per_task_ms": 168506.08666666667, "avg_output_tokens_per_task": 4247.114666666666, "last_run_timestamp": "2026-10-24T13:06:52.407Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.7416665, "accuracy_std_dev": 0.03140250343921645, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012469425000000001, "avg_duration_per_task_ms": 90319.205, "avg_output_tokens_per_task": 10893.135, "last_run_timestamp": "2026-10-24T12:16:35.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7414814444444444, "accuracy_std_dev": 0.014665955839677083, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11785513555555556, "avg_duration_per_task_ms": 351306.1688888889, "avg_output_tokens_per_task": 9806.686666666666, "last_run_timestamp": "2026-10-24T20:00:28.580Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7413334, "accuracy_std_dev": 0.02578533261061415, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.064516224, "avg_duration_per_task_ms": 309719.048, "avg_output_tokens_per_task": 8304.048, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7412120909090909, "accuracy_std_dev": 0.033492690704511933, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.027686032727272727, "avg_duration_per_task_ms": 442749.78545454546, "avg_output_tokens_per_task": 14610.476363636364, "last_run_timestamp": "2026-10-17T13:55:17.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7411110000000001, "accuracy_std_dev": 0.02078704487575536, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06692624, "avg_duration_per_task_ms": 156822.04, "avg_output_tokens_per_task": 7280.366666666667, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7410191879699248, "accuracy_std_dev": 0.023857300276526983, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.22161520300751877, "avg_duration_per_task_ms": 375984.74686716794, "avg_output_tokens_per_task": 9942.047619047618, "last_run_timestamp": "2026-10-24T12:46:10.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.74083325, "accuracy_std_dev": 0.030080976362769567, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05869578, "avg_duration_per_task_ms": 714580.6575, "avg_output_tokens_per_task": 15701.38, "last_run_timestamp": "2026-10-24T19:50:11.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7405555, "accuracy_std_dev": 0.01855080944101005, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.0133585, "avg_duration_per_task_ms": 101370.275, "avg_output_tokens_per_task": 10830.071666666667, "last_run_timestamp": "2026-10-24T11:56:06.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7403703333333334, "accuracy_std_dev": 0.030325141571529955, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3409674066666667, "avg_duration_per_task_ms": 128710.22, "avg_output_tokens_per_task": 5246.528888888889, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7403685728643216, "accuracy_std_dev": 0.02451522166373828, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.05593858793969849, "avg_duration_per_task_ms": 169768.54773869348, "avg_output_tokens_per_task": 30975.941373534337, "last_run_timestamp": "2026-10-24T12:06:17.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400002, "accuracy_std_dev": 0.01264900523203309, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034869184, "avg_duration_per_task_ms": 129959.044, "avg_output_tokens_per_task": 16349.368, "last_run_timestamp": "2026-10-03T08:21:01.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7400000999999999, "accuracy_std_dev": 0.02022089128821974, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.37108522400000005, "avg_duration_per_task_ms": 184691.406, "avg_output_tokens_per_task": 10675.186, "last_run_timestamp": "2026-10-24T12:36:29.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08086026, "avg_duration_per_task_ms": 522771.74, "avg_output_tokens_per_task": 13545.48, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.74, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.024783569999999998, "avg_duration_per_task_ms": 140465.05, "avg_output_tokens_per_task": 20671.25, "last_run_timestamp": "2026-10-24T12:32:09.787Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.74, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02650346, "avg_duration_per_task_ms": 245657.46, "avg_output_tokens_per_task": 33745.71, "last_run_timestamp": "2026-10-24T12:08:32.744Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.74, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02533324, "avg_duration_per_task_ms": 117353.99, "avg_output_tokens_per_task": 11491.8, "last_run_timestamp": "2026-10-24T11:51:14.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08715152, "avg_duration_per_task_ms": 69751.7, "avg_output_tokens_per_task": 8640, "last_run_timestamp": "2026-10-24T11:33:58.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.74, "accuracy_std_dev": 0.044555500139339316, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 1.0935073933333332, "avg_duration_per_task_ms": 295552.94, "avg_output_tokens_per_task": 22860.126666666667, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.74, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01884005, "avg_duration_per_task_ms": 67349.35, "avg_output_tokens_per_task": 22014.7, "last_run_timestamp": "2026-10-24T11:48:47.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.026460249999999998, "avg_duration_per_task_ms": 73471.9, "avg_output_tokens_per_task": 4407.31, "last_run_timestamp": "2026-10-24T13:04:21.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7399999, "accuracy_std_dev": 0.028906131607152153, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.34949705000000003, "avg_duration_per_task_ms": 220413.082, "avg_output_tokens_per_task": 10856.716, "last_run_timestamp": "2026-10-24T12:39:24.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7399998571428572, "accuracy_std_dev": 0.023094052007442304, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01886730285714286, "avg_duration_per_task_ms": 206247.68, "avg_output_tokens_per_task": 4550.928571428572, "last_run_timestamp": "2026-10-10T11:50:34.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7395835, "accuracy_std_dev": 0.022012461748973016, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.042535722500000005, "avg_duration_per_task_ms": 134957.37, "avg_output_tokens_per_task": 25971.215, "last_run_timestamp": "2026-10-23T03:54:04.118Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7395367713004484, "accuracy_std_dev": 0.016058910754384415, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.017859735426008966, "avg_duration_per_task_ms": 83110.88116591929, "avg_output_tokens_per_task": 4085.5941704035877, "last_run_timestamp": "2026-10-24T12:42:47.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7393333, "accuracy_std_dev": 0.04912106993144592, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04842688399999999, "avg_duration_per_task_ms": 284804.346, "avg_output_tokens_per_task": 14992.81, "last_run_timestamp": "2026-10-24T12:17:04.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.7392593333333333, "accuracy_std_dev": 0.036167601020188705, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04373026666666667, "avg_duration_per_task_ms": 581663.7222222222, "avg_output_tokens_per_task": 9145.893333333333, "last_run_timestamp": "2026-10-24T19:13:45.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7392592222222222, "accuracy_std_dev": 0.04548160272700695, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16905808444444445, "avg_duration_per_task_ms": 289473.89555555553, "avg_output_tokens_per_task": 9765.786666666667, "last_run_timestamp": "2026-10-24T11:42:23.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7392592222222222, "accuracy_std_dev": 0.03955854406875342, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12468993111111111, "avg_duration_per_task_ms": 675089.5488888889, "avg_output_tokens_per_task": 35068.42222222222, "last_run_timestamp": "2026-10-24T11:39:42.358Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7392592222222222, "accuracy_std_dev": 0.02938791032145241, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1616571777777778, "avg_duration_per_task_ms": 223287.44222222222, "avg_output_tokens_per_task": 9621.846666666666, "last_run_timestamp": "2026-10-24T19:37:14.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.73916675, "accuracy_std_dev": 0.029521604613189638, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.079499585, "avg_duration_per_task_ms": 501652.2525, "avg_output_tokens_per_task": 11995.7875, "last_run_timestamp": "2026-10-24T23:08:37.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.738667, "accuracy_std_dev": 0.020396078054371124, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019569312, "avg_duration_per_task_ms": 507361.484, "avg_output_tokens_per_task": 14742.804, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7386668, "accuracy_std_dev": 0.03166503013357166, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009092168000000001, "avg_duration_per_task_ms": 101935.708, "avg_output_tokens_per_task": 5027.116, "last_run_timestamp": "2026-10-17T11:17:02.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.017992647486688756, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3103866155555556, "avg_duration_per_task_ms": 191198.25333333333, "avg_output_tokens_per_task": 10879.702222222222, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7384824959349594, "accuracy_std_dev": 0.030529621622286764, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.012032963414634147, "avg_duration_per_task_ms": 61127.26829268293, "avg_output_tokens_per_task": 4200.292682926829, "last_run_timestamp": "2026-10-24T12:38:23.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7383335, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08170183999999998, "avg_duration_per_task_ms": 327224.56, "avg_output_tokens_per_task": 8775.96, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.73833325, "accuracy_std_dev": 0.02153795745161318, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.34754801999999996, "avg_duration_per_task_ms": 197375.055, "avg_output_tokens_per_task": 8811.4275, "last_run_timestamp": "2026-10-24T12:17:34.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7382021797752809, "accuracy_std_dev": 0.021880590311850887, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.020498339325842697, "avg_duration_per_task_ms": 307841.11235955055, "avg_output_tokens_per_task": 10994.101123595505, "last_run_timestamp": "2026-10-17T11:09:59.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.73809525, "accuracy_std_dev": 0.03175464476067312, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.09706365848214285, "avg_duration_per_task_ms": 342421.30803571426, "avg_output_tokens_per_task": 11142.66517857143, "last_run_timestamp": "2026-10-24T20:22:47.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7380950401002507, "accuracy_std_dev": 0.022649397541656617, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.053091303258145366, "avg_duration_per_task_ms": 777194.3383458647, "avg_output_tokens_per_task": 14178.676691729323, "last_run_timestamp": "2026-10-24T19:40:41.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7377776666666667, "accuracy_std_dev": 0.017777750001367207, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014543953333333333, "avg_duration_per_task_ms": 73473.39777777778, "avg_output_tokens_per_task": 4452.611111111111, "last_run_timestamp": "2026-10-24T12:18:30.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7375756363636364, "accuracy_std_dev": 0.024498683461000814, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.008059856363636365, "avg_duration_per_task_ms": 393755.9490909091, "avg_output_tokens_per_task": 14461.965454545454, "last_run_timestamp": "2026-10-17T12:09:02.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7374528407310704, "accuracy_std_dev": 0.027468652356798965, "run_count": 23, "total_questions": 1149, "avg_cost_per_task_usd": 0.12826692602262835, "avg_duration_per_task_ms": 160716.89469103568, "avg_output_tokens_per_task": 9299.472584856398, "last_run_timestamp": "2026-10-24T19:48:18.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7373334, "accuracy_std_dev": 0.020482933819157834, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039714615999999994, "avg_duration_per_task_ms": 174773.632, "avg_output_tokens_per_task": 27822.08, "last_run_timestamp": "2026-10-21T12:36:51.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.7371428571428571, "accuracy_std_dev": 0.023866691370602513, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.022481400000000002, "avg_duration_per_task_ms": 144173.6, "avg_output_tokens_per_task": 16498.17142857143, "last_run_timestamp": "2026-10-24T12:33:37.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.737037, "accuracy_std_dev": 0.03543182483775474, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09546396444444445, "avg_duration_per_task_ms": 226782.94222222222, "avg_output_tokens_per_task": 12230.953333333333, "last_run_timestamp": "2026-10-24T19:51:30.269Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.737036888888889, "accuracy_std_dev": 0.020872504739459674, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06686444222222222, "avg_duration_per_task_ms": 272840.34222222224, "avg_output_tokens_per_task": 12781.422222222222, "last_run_timestamp": "2026-10-24T20:26:13.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2879983000000002, "avg_duration_per_task_ms": 310547.73, "avg_output_tokens_per_task": 22913.4, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7366668, "accuracy_std_dev": 0.022900761755015906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.063241744, "avg_duration_per_task_ms": 1530537.112, "avg_output_tokens_per_task": 25222.716, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02539905, "avg_duration_per_task_ms": 216631.23, "avg_output_tokens_per_task": 14336.27, "last_run_timestamp": "2026-10-24T12:02:36.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7365772046979866, "accuracy_std_dev": 0.061530610928569984, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.15594882046979863, "avg_duration_per_task_ms": 1382559.5033557047, "avg_output_tokens_per_task": 28131.454697986577, "last_run_timestamp": "2026-10-24T20:45:53.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7365591209677419, "accuracy_std_dev": 0.016931426040354632, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.23607216935483874, "avg_duration_per_task_ms": 378114.625, "avg_output_tokens_per_task": 11505.427419354839, "last_run_timestamp": "2026-10-24T11:40:39.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.7362963333333333, "accuracy_std_dev": 0.01605880825106412, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07133713555555556, "avg_duration_per_task_ms": 247650.13333333333, "avg_output_tokens_per_task": 13143.48, "last_run_timestamp": "2026-10-24T19:41:01.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7361904285714286, "accuracy_std_dev": 0.035964376429450086, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.024463299999999997, "avg_duration_per_task_ms": 153502.6, "avg_output_tokens_per_task": 16394.091428571428, "last_run_timestamp": "2026-10-24T11:47:20.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7361111666666666, "accuracy_std_dev": 0.029994961512542214, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02623104, "avg_duration_per_task_ms": 96319.25333333333, "avg_output_tokens_per_task": 9025.273333333333, "last_run_timestamp": "2026-10-24T19:08:17.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7361111666666666, "accuracy_std_dev": 0.03656122127754426, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.025397581666666662, "avg_duration_per_task_ms": 117939.91666666667, "avg_output_tokens_per_task": 8683.921666666667, "last_run_timestamp": "2026-10-24T19:07:29.704Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.736, "accuracy_std_dev": 0.02908991425219403, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.035272232, "avg_duration_per_task_ms": 341115.712, "avg_output_tokens_per_task": 8366.916, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.735926, "accuracy_std_dev": 0.03280219513318514, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08689671777777778, "avg_duration_per_task_ms": 176906.36444444445, "avg_output_tokens_per_task": 10416.388888888889, "last_run_timestamp": "2026-10-24T19:53:44.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7358025185185185, "accuracy_std_dev": 0.032868308079781576, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.35590564518518514, "avg_duration_per_task_ms": 206939.88074074074, "avg_output_tokens_per_task": 10319.86, "last_run_timestamp": "2026-10-24T12:18:44.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7355555833333334, "accuracy_std_dev": 0.043489812913406, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08088585999999999, "avg_duration_per_task_ms": 169768.16333333333, "avg_output_tokens_per_task": 8239.883333333333, "last_run_timestamp": "2026-10-24T19:35:51.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7355555, "accuracy_std_dev": 0.026293678535280442, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07820470500000001, "avg_duration_per_task_ms": 167526.81, "avg_output_tokens_per_task": 8785.875, "last_run_timestamp": "2026-10-24T19:32:01.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7354165, "accuracy_std_dev": 0.0429125354005097, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.006874592499999999, "avg_duration_per_task_ms": 421319.53, "avg_output_tokens_per_task": 14336.97, "last_run_timestamp": "2026-10-10T15:31:23.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7351191205357143, "accuracy_std_dev": 0.049992352433912764, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.30342583258928574, "avg_duration_per_task_ms": 387074.1026785714, "avg_output_tokens_per_task": 11534.020089285714, "last_run_timestamp": "2026-10-24T13:33:47.625Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.735, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012519140000000002, "avg_duration_per_task_ms": 95304.945, "avg_output_tokens_per_task": 10824.09, "last_run_timestamp": "2026-10-24T11:50:10.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7349997500000001, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12295462999999998, "avg_duration_per_task_ms": 89986.35, "avg_output_tokens_per_task": 23350.75, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7349570573065901, "accuracy_std_dev": 0.02405249324595975, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.067489111747851, "avg_duration_per_task_ms": 124543.75071633238, "avg_output_tokens_per_task": 4309.329512893983, "last_run_timestamp": "2026-10-24T12:40:32.142Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7349569369627508, "accuracy_std_dev": 0.02758952341386278, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.13143638108882524, "avg_duration_per_task_ms": 67067.49856733525, "avg_output_tokens_per_task": 9819, "last_run_timestamp": "2026-10-24T18:49:16.415Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7348148888888889, "accuracy_std_dev": 0.026439290919406052, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05043111333333333, "avg_duration_per_task_ms": 363684.43555555557, "avg_output_tokens_per_task": 10090.468888888889, "last_run_timestamp": "2026-10-24T20:19:38.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7348008584905661, "accuracy_std_dev": 0.031872568589016145, "run_count": 13, "total_questions": 636, "avg_cost_per_task_usd": 0.06298322641509434, "avg_duration_per_task_ms": 126873.33333333333, "avg_output_tokens_per_task": 8176.9606918239, "last_run_timestamp": "2026-10-24T19:13:30.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.73458325, "accuracy_std_dev": 0.027785801480927293, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0151816525, "avg_duration_per_task_ms": 268037.2675, "avg_output_tokens_per_task": 21770.385, "last_run_timestamp": "2026-10-24T12:56:13.097Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7344444166666667, "accuracy_std_dev": 0.025361547399420575, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.3672292141666667, "avg_duration_per_task_ms": 228755.93166666667, "avg_output_tokens_per_task": 10889.619166666667, "last_run_timestamp": "2026-10-24T12:16:43.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.7341665, "accuracy_std_dev": 0.02066601477668108, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012598340000000001, "avg_duration_per_task_ms": 74617.86, "avg_output_tokens_per_task": 10550.06, "last_run_timestamp": "2026-10-24T11:45:42.778Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7340742222222222, "accuracy_std_dev": 0.027296881245453555, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.017404215555555554, "avg_duration_per_task_ms": 575071.7644444444, "avg_output_tokens_per_task": 18088.03111111111, "last_run_timestamp": "2026-10-24T11:35:52.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7340738888888889, "accuracy_std_dev": 0.03993117810668319, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09019846222222223, "avg_duration_per_task_ms": 227703.9688888889, "avg_output_tokens_per_task": 10092.924444444445, "last_run_timestamp": "2026-10-24T20:31:27.967Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7338888333333333, "accuracy_std_dev": 0.030937109644226465, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03372703333333333, "avg_duration_per_task_ms": 429862.66333333333, "avg_output_tokens_per_task": 9026.426666666666, "last_run_timestamp": "2026-10-24T20:40:57.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.733796, "accuracy_std_dev": 0.0059017228557995535, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.20467558333333333, "avg_duration_per_task_ms": 370954.7361111111, "avg_output_tokens_per_task": 23212.9375, "last_run_timestamp": "2026-10-24T19:18:46.938Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7334173324937028, "accuracy_std_dev": 0.036811483722601776, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.08725528211586901, "avg_duration_per_task_ms": 292129.2695214106, "avg_output_tokens_per_task": 14240.488664987406, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7333336666666668, "accuracy_std_dev": 0.03771236166328251, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.44470873333333333, "avg_duration_per_task_ms": 130557.96, "avg_output_tokens_per_task": 20380.893333333333, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.7333335, "accuracy_std_dev": 0.033333500000000016, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018280730000000002, "avg_duration_per_task_ms": 88642.31, "avg_output_tokens_per_task": 11899.17, "last_run_timestamp": "2026-10-24T11:54:38.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08394800999999999, "avg_duration_per_task_ms": 134148.76, "avg_output_tokens_per_task": 8723.94, "last_run_timestamp": "2026-10-17T19:55:39.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.047618709999999995, "avg_duration_per_task_ms": 115029.78, "avg_output_tokens_per_task": 15522.32, "last_run_timestamp": "2026-10-24T12:31:48.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.733333375, "accuracy_std_dev": 0.03507936636378106, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07472340000000001, "avg_duration_per_task_ms": 205272.835, "avg_output_tokens_per_task": 11169.0925, "last_run_timestamp": "2026-10-24T19:35:24.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05760637824832321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04898614, "avg_duration_per_task_ms": 106829.63333333333, "avg_output_tokens_per_task": 9897.326666666666, "last_run_timestamp": "2026-10-10T11:51:48.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.73333325, "accuracy_std_dev": 0.02357013176007932, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04045966999999999, "avg_duration_per_task_ms": 165902.775, "avg_output_tokens_per_task": 35276.36, "last_run_timestamp": "2026-10-24T12:26:22.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.73333325, "accuracy_std_dev": 0.03377488109064932, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.30121887833333333, "avg_duration_per_task_ms": 325660.4733333333, "avg_output_tokens_per_task": 9348.415, "last_run_timestamp": "2026-10-24T13:17:19.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.73333325, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17039939499999998, "avg_duration_per_task_ms": 114141.445, "avg_output_tokens_per_task": 5083.04, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01490002, "avg_duration_per_task_ms": 87918.52, "avg_output_tokens_per_task": 12302.3, "last_run_timestamp": "2026-10-24T11:33:56.850Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.038872460000000005, "avg_duration_per_task_ms": 158714.4, "avg_output_tokens_per_task": 14542.76, "last_run_timestamp": "2026-10-24T13:34:38.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02595887, "avg_duration_per_task_ms": 77664, "avg_output_tokens_per_task": 16596.59, "last_run_timestamp": "2026-10-24T12:17:54.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7332197098976109, "accuracy_std_dev": 0.022906695901829432, "run_count": 6, "total_questions": 293, "avg_cost_per_task_usd": 0.04657865870307167, "avg_duration_per_task_ms": 71132.01023890785, "avg_output_tokens_per_task": 19231.02389078498, "last_run_timestamp": "2026-10-24T12:19:39.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7331107060133629, "accuracy_std_dev": 0.030637914267565063, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.06578883518930957, "avg_duration_per_task_ms": 155774.63919821827, "avg_output_tokens_per_task": 11358.256124721604, "last_run_timestamp": "2026-10-24T19:41:50.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02503509736024079, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06745117142857142, "avg_duration_per_task_ms": 76562.92857142857, "avg_output_tokens_per_task": 8496.594285714285, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7327847422303474, "accuracy_std_dev": 0.025507937405721676, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.35576682998171844, "avg_duration_per_task_ms": 402866.0219378428, "avg_output_tokens_per_task": 9582.784277879342, "last_run_timestamp": "2026-10-24T12:29:09.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7325, "accuracy_std_dev": 0.04596937990662913, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012367795, "avg_duration_per_task_ms": 953069.875, "avg_output_tokens_per_task": 22737, "last_run_timestamp": "2026-10-17T10:32:07.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.7325, "accuracy_std_dev": 0.029474565306379014, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08144697000000001, "avg_duration_per_task_ms": 228561.56, "avg_output_tokens_per_task": 29757.14, "last_run_timestamp": "2026-10-24T12:23:33.838Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7318182727272727, "accuracy_std_dev": 0.040982052557610026, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08459119272727272, "avg_duration_per_task_ms": 333975.78909090906, "avg_output_tokens_per_task": 9334.785454545454, "last_run_timestamp": "2026-10-24T19:37:38.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73166675, "accuracy_std_dev": 0.015184321229067165, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.29879598, "avg_duration_per_task_ms": 177541.565, "avg_output_tokens_per_task": 10870.785, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7315151818181818, "accuracy_std_dev": 0.012091042761393707, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.23347585818181818, "avg_duration_per_task_ms": 257016.06181818183, "avg_output_tokens_per_task": 10120.247272727273, "last_run_timestamp": "2026-10-24T12:33:16.806Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7312639731543625, "accuracy_std_dev": 0.031328778264981705, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.10314290604026845, "avg_duration_per_task_ms": 101786.30704697987, "avg_output_tokens_per_task": 7660.4093959731545, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7311113333333332, "accuracy_std_dev": 0.0332591470759883, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03472362, "avg_duration_per_task_ms": 267399.06, "avg_output_tokens_per_task": 33925.37333333334, "last_run_timestamp": "2026-10-23T07:22:37.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7311113333333332, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10536965333333334, "avg_duration_per_task_ms": 88766.42666666667, "avg_output_tokens_per_task": 4017.0533333333333, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.022662199996175744, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3988358066666667, "avg_duration_per_task_ms": 172929.82666666666, "avg_output_tokens_per_task": 27148.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.7309522857142857, "accuracy_std_dev": 0.01941313112092864, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04753026285714287, "avg_duration_per_task_ms": 161306.4342857143, "avg_output_tokens_per_task": 28780.68857142857, "last_run_timestamp": "2026-10-24T12:41:08.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7306985367647059, "accuracy_std_dev": 0.04536902430892472, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.025314847426470583, "avg_duration_per_task_ms": 110991.27205882352, "avg_output_tokens_per_task": 8956.389705882353, "last_run_timestamp": "2026-10-24T19:18:47.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.7306399292929292, "accuracy_std_dev": 0.010748499999999994, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.08618277777777777, "avg_duration_per_task_ms": 204214.55555555556, "avg_output_tokens_per_task": 8369.424242424242, "last_run_timestamp": "2026-10-24T12:28:29.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.7304762857142857, "accuracy_std_dev": 0.028026480838736798, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04970431714285714, "avg_duration_per_task_ms": 453038.12, "avg_output_tokens_per_task": 9603.891428571429, "last_run_timestamp": "2026-10-24T20:04:11.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7303704444444444, "accuracy_std_dev": 0.039013983635817075, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09884042444444445, "avg_duration_per_task_ms": 220383.57555555555, "avg_output_tokens_per_task": 11667.057777777778, "last_run_timestamp": "2026-10-24T20:41:03.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7303704444444444, "accuracy_std_dev": 0.02882252301050966, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05139691333333334, "avg_duration_per_task_ms": 265789.9711111111, "avg_output_tokens_per_task": 10843.32, "last_run_timestamp": "2026-10-24T19:37:52.115Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7303473382084096, "accuracy_std_dev": 0.033471576963840254, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.07826806032906765, "avg_duration_per_task_ms": 206680.5575868373, "avg_output_tokens_per_task": 8422.92870201097, "last_run_timestamp": "2026-10-24T19:29:28.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.73, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023320379999999998, "avg_duration_per_task_ms": 139023.54, "avg_output_tokens_per_task": 12073.4, "last_run_timestamp": "2026-10-24T12:04:17.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.73, "accuracy_std_dev": 0.023332999999999993, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0186985, "avg_duration_per_task_ms": 114277.39, "avg_output_tokens_per_task": 11475.52, "last_run_timestamp": "2026-10-24T11:36:27.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.73, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014710339999999999, "avg_duration_per_task_ms": 195605.73, "avg_output_tokens_per_task": 13552.09, "last_run_timestamp": "2026-10-24T11:46:43.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.73, "accuracy_std_dev": 0.02860989210046065, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08111631999999999, "avg_duration_per_task_ms": 225903.98, "avg_output_tokens_per_task": 8790.808333333332, "last_run_timestamp": "2026-10-24T19:45:17.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.73, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01638479, "avg_duration_per_task_ms": 69570.4, "avg_output_tokens_per_task": 4309.49, "last_run_timestamp": "2026-10-24T12:32:51.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.73, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02344918, "avg_duration_per_task_ms": 35583.1, "avg_output_tokens_per_task": 4177.41, "last_run_timestamp": "2026-10-24T11:37:33.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7294444166666667, "accuracy_std_dev": 0.03271169279849418, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.33465930833333335, "avg_duration_per_task_ms": 207392.885, "avg_output_tokens_per_task": 10389.821666666667, "last_run_timestamp": "2026-10-24T12:22:38.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7290908181818182, "accuracy_std_dev": 0.028606911904192237, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.2888466727272727, "avg_duration_per_task_ms": 345375.0236363636, "avg_output_tokens_per_task": 10075.607272727273, "last_run_timestamp": "2026-10-24T12:45:08.706Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7285714285714285, "accuracy_std_dev": 0.026598516660344282, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04723099714285714, "avg_duration_per_task_ms": 142250.88571428572, "avg_output_tokens_per_task": 24794.32857142857, "last_run_timestamp": "2026-10-24T13:07:38.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7285712857142858, "accuracy_std_dev": 0.024871537523352083, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03253488857142857, "avg_duration_per_task_ms": 92457.76285714285, "avg_output_tokens_per_task": 23099.80285714286, "last_run_timestamp": "2026-10-24T11:48:04.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7284945564516129, "accuracy_std_dev": 0.025519129645032956, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.06162772983870967, "avg_duration_per_task_ms": 509436.8830645161, "avg_output_tokens_per_task": 14294.16935483871, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.72833325, "accuracy_std_dev": 0.029829774773216668, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.35398706500000005, "avg_duration_per_task_ms": 238402.66166666665, "avg_output_tokens_per_task": 10886.34, "last_run_timestamp": "2026-10-24T11:45:41.975Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.72833325, "accuracy_std_dev": 0.03443989530961877, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09767092499999999, "avg_duration_per_task_ms": 823173.835, "avg_output_tokens_per_task": 9664.385, "last_run_timestamp": "2026-10-24T20:25:57.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.7276631288659794, "accuracy_std_dev": 0.019224116824642454, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.07521926804123712, "avg_duration_per_task_ms": 73859.24742268042, "avg_output_tokens_per_task": 8316.994845360825, "last_run_timestamp": "2026-10-24T11:42:23.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7275, "accuracy_std_dev": 0.022220636056377873, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06640188, "avg_duration_per_task_ms": 199646.51, "avg_output_tokens_per_task": 9529.36, "last_run_timestamp": "2026-10-24T19:34:44.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7272047264573992, "accuracy_std_dev": 0.08169417811313268, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.009946286995515695, "avg_duration_per_task_ms": 180161.19058295965, "avg_output_tokens_per_task": 4234.017937219731, "last_run_timestamp": "2026-10-24T12:25:45.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21341358, "avg_duration_per_task_ms": 101630.84, "avg_output_tokens_per_task": 8827.48, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11300302000000001, "avg_duration_per_task_ms": 353321.78, "avg_output_tokens_per_task": 3658.14, "last_run_timestamp": "2026-10-03T00:00:06.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.047453712340436424, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011438460000000001, "avg_duration_per_task_ms": 86937.99333333333, "avg_output_tokens_per_task": 10682.4, "last_run_timestamp": "2026-10-24T11:51:31.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02419578, "avg_duration_per_task_ms": 133541.80666666667, "avg_output_tokens_per_task": 11460.326666666666, "last_run_timestamp": "2026-10-24T12:24:12.065Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.7266665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01900955, "avg_duration_per_task_ms": 214869.47, "avg_output_tokens_per_task": 11727.86, "last_run_timestamp": "2026-10-24T12:08:01.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018476000000000003, "avg_duration_per_task_ms": 104934.72, "avg_output_tokens_per_task": 10917.83, "last_run_timestamp": "2026-10-24T11:48:43.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7266665, "accuracy_std_dev": 0.03162256578378802, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.325251515, "avg_duration_per_task_ms": 113433.705, "avg_output_tokens_per_task": 10925.445, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.7263158421052631, "accuracy_std_dev": 0.007406999999999997, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.013109768421052631, "avg_duration_per_task_ms": 155953.90526315788, "avg_output_tokens_per_task": 8164.357894736842, "last_run_timestamp": "2026-10-24T13:07:13.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7261505656565657, "accuracy_std_dev": 0.07912140749203615, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.04286990067340068, "avg_duration_per_task_ms": 271792.98484848486, "avg_output_tokens_per_task": 24134.688552188552, "last_run_timestamp": "2026-10-24T19:21:10.232Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7261110833333333, "accuracy_std_dev": 0.02574195007977683, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08434359333333334, "avg_duration_per_task_ms": 650589.2116666667, "avg_output_tokens_per_task": 9213.296666666667, "last_run_timestamp": "2026-10-24T19:18:30.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7260606363636364, "accuracy_std_dev": 0.037546404958694955, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.009626858181818184, "avg_duration_per_task_ms": 366334.8836363636, "avg_output_tokens_per_task": 14262.687272727273, "last_run_timestamp": "2026-10-17T13:13:33.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7259886915254237, "accuracy_std_dev": 0.03421740126220705, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.01908770508474576, "avg_duration_per_task_ms": 310174.90847457625, "avg_output_tokens_per_task": 6632.271186440678, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7259258888888889, "accuracy_std_dev": 0.02618905414457831, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12695806222222222, "avg_duration_per_task_ms": 251909.17111111112, "avg_output_tokens_per_task": 10238.07111111111, "last_run_timestamp": "2026-10-24T22:56:39.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7256409230769231, "accuracy_std_dev": 0.024647466274717048, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.07219294769230769, "avg_duration_per_task_ms": 219921.15384615384, "avg_output_tokens_per_task": 8518.82, "last_run_timestamp": "2026-10-24T19:52:00.596Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7251852222222223, "accuracy_std_dev": 0.024500620077313125, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05734841777777778, "avg_duration_per_task_ms": 328949.4666666667, "avg_output_tokens_per_task": 9955.037777777778, "last_run_timestamp": "2026-10-24T20:09:59.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7251850000000001, "accuracy_std_dev": 0.03496421058739923, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05816458666666668, "avg_duration_per_task_ms": 362209.84222222224, "avg_output_tokens_per_task": 11818.435555555556, "last_run_timestamp": "2026-10-24T20:11:31.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7250074958979034, "accuracy_std_dev": 0.02896808645605747, "run_count": 22, "total_questions": 1097, "avg_cost_per_task_usd": 0.4289184184138559, "avg_duration_per_task_ms": 233970.86873290793, "avg_output_tokens_per_task": 10583.862351868733, "last_run_timestamp": "2026-10-24T12:16:44.774Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7249999166666666, "accuracy_std_dev": 0.035603221088965, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08727709499999999, "avg_duration_per_task_ms": 139180.04, "avg_output_tokens_per_task": 9126.01, "last_run_timestamp": "2026-10-24T18:58:26.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7244622983870967, "accuracy_std_dev": 0.027938338852551706, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.061894955645161294, "avg_duration_per_task_ms": 511647.25, "avg_output_tokens_per_task": 14459.54435483871, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7242855714285714, "accuracy_std_dev": 0.024734242792748124, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04387056285714286, "avg_duration_per_task_ms": 165104.78285714285, "avg_output_tokens_per_task": 29616.194285714286, "last_run_timestamp": "2026-10-24T12:35:19.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7241666250000001, "accuracy_std_dev": 0.018540625440755097, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1069103825, "avg_duration_per_task_ms": 322528.1575, "avg_output_tokens_per_task": 14528.2375, "last_run_timestamp": "2026-10-17T19:16:53.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.724, "accuracy_std_dev": 0.031721020021430596, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.053489452, "avg_duration_per_task_ms": 274950.848, "avg_output_tokens_per_task": 8530.916, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.72333325, "accuracy_std_dev": 0.01374358436462263, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011429815, "avg_duration_per_task_ms": 101515.345, "avg_output_tokens_per_task": 5183.985, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7231578947368421, "accuracy_std_dev": 0.03181579445709768, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.06943197789473686, "avg_duration_per_task_ms": 118880.57368421053, "avg_output_tokens_per_task": 8790.421052631578, "last_run_timestamp": "2026-10-24T19:51:31.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.722963, "accuracy_std_dev": 0.024567656927124495, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.2753419711111111, "avg_duration_per_task_ms": 183403.79555555555, "avg_output_tokens_per_task": 11487.52, "last_run_timestamp": "2026-10-24T12:33:54.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.722963, "accuracy_std_dev": 0.015748325844002865, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05190752666666666, "avg_duration_per_task_ms": 384896.1533333333, "avg_output_tokens_per_task": 9409.375555555556, "last_run_timestamp": "2026-10-24T20:09:28.564Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.7226663999999999, "accuracy_std_dev": 0.028782797331739687, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04632364, "avg_duration_per_task_ms": 161759.348, "avg_output_tokens_per_task": 8208.956, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7224079799331103, "accuracy_std_dev": 0.027594435979393265, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.046122013377926416, "avg_duration_per_task_ms": 197207.33444816055, "avg_output_tokens_per_task": 9716.61872909699, "last_run_timestamp": "2026-10-24T19:18:28.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7223809999999999, "accuracy_std_dev": 0.03270676301623258, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.1190975942857143, "avg_duration_per_task_ms": 133109.795, "avg_output_tokens_per_task": 8591.442857142858, "last_run_timestamp": "2026-10-24T19:56:04.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.7223148931552588, "accuracy_std_dev": 0.03960298431691171, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.025822385642737894, "avg_duration_per_task_ms": 283083.48080133554, "avg_output_tokens_per_task": 7842.495826377295, "last_run_timestamp": "2026-10-24T12:07:12.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.72166675, "accuracy_std_dev": 0.012801898733683979, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01389351, "avg_duration_per_task_ms": 148770.74, "avg_output_tokens_per_task": 13265.9, "last_run_timestamp": "2026-10-24T12:22:53.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7206667999999999, "accuracy_std_dev": 0.037882852175621616, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.29679829, "avg_duration_per_task_ms": 498320.864, "avg_output_tokens_per_task": 9522.336, "last_run_timestamp": "2026-10-24T22:09:08.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7205555, "accuracy_std_dev": 0.020268346362000044, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08116760333333334, "avg_duration_per_task_ms": 238268.15166666667, "avg_output_tokens_per_task": 9240.011666666667, "last_run_timestamp": "2026-10-24T19:37:32.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7205384141414142, "accuracy_std_dev": 0.051112796809483045, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 1.6761988181818182, "avg_duration_per_task_ms": 313695.66161616164, "avg_output_tokens_per_task": 23404.97474747475, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.7200002499999999, "accuracy_std_dev": 0.016329931619192377, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.005028675, "avg_duration_per_task_ms": 140463.585, "avg_output_tokens_per_task": 11883.17, "last_run_timestamp": "2026-10-24T12:43:45.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.72, "accuracy_std_dev": 0.03231762505197435, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.36211610499999997, "avg_duration_per_task_ms": 150104.23, "avg_output_tokens_per_task": 23123.4, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.72, "accuracy_std_dev": 0.03165460057329021, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.028238483636363637, "avg_duration_per_task_ms": 217753.1509090909, "avg_output_tokens_per_task": 9627.723636363637, "last_run_timestamp": "2026-10-24T19:14:23.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.053717240000000006, "avg_duration_per_task_ms": 263357.62, "avg_output_tokens_per_task": 8791.84, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.72, "accuracy_std_dev": 0.05409184233782393, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11944148833333335, "avg_duration_per_task_ms": 138004.31, "avg_output_tokens_per_task": 9585.921666666667, "last_run_timestamp": "2026-10-24T19:35:31.077Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7199998461538462, "accuracy_std_dev": 0.060595320121208315, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.28468824153846156, "avg_duration_per_task_ms": 424519.81384615385, "avg_output_tokens_per_task": 10050.18923076923, "last_run_timestamp": "2026-10-24T12:30:07.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7199996666666667, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09322211333333333, "avg_duration_per_task_ms": 149908.11333333334, "avg_output_tokens_per_task": 49346.81333333333, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.719858, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.0657194255319149, "avg_duration_per_task_ms": 49852.255319148935, "avg_output_tokens_per_task": 5715.191489361702, "last_run_timestamp": "2026-10-24T12:16:29.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7193332, "accuracy_std_dev": 0.03143937799893632, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045152332, "avg_duration_per_task_ms": 326196.928, "avg_output_tokens_per_task": 13427.836, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7188551919191919, "accuracy_std_dev": 0.027525589315898408, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.29904142424242425, "avg_duration_per_task_ms": 154139.7323232323, "avg_output_tokens_per_task": 5036.570707070707, "last_run_timestamp": "2026-10-01T19:05:12.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.7185185555555555, "accuracy_std_dev": 0.0497675821798155, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07125336888888889, "avg_duration_per_task_ms": 245562.12, "avg_output_tokens_per_task": 8258.706666666667, "last_run_timestamp": "2026-10-24T19:39:33.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7185183333333334, "accuracy_std_dev": 0.033815486586933986, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.24051653333333337, "avg_duration_per_task_ms": 300854.7, "avg_output_tokens_per_task": 21246.093333333334, "last_run_timestamp": "2026-10-24T12:24:27.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7177777777777777, "accuracy_std_dev": 0.018986661986643713, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.03833726666666666, "avg_duration_per_task_ms": 195653.57888888888, "avg_output_tokens_per_task": 9865.436666666666, "last_run_timestamp": "2026-10-24T19:52:24.184Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.7177776666666666, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10180220666666666, "avg_duration_per_task_ms": 128107.32666666666, "avg_output_tokens_per_task": 16602.8, "last_run_timestamp": "2026-10-24T12:39:45.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7173648633879781, "accuracy_std_dev": 0.031638898989055704, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.0967067795992714, "avg_duration_per_task_ms": 190644.16393442624, "avg_output_tokens_per_task": 10605.225865209472, "last_run_timestamp": "2026-10-17T19:53:39.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7171491336302895, "accuracy_std_dev": 0.026956007306143996, "run_count": 18, "total_questions": 898, "avg_cost_per_task_usd": 0.10596277951002227, "avg_duration_per_task_ms": 105231.55345211581, "avg_output_tokens_per_task": 9573.741648106905, "last_run_timestamp": "2026-10-24T19:32:20.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.7170371111111111, "accuracy_std_dev": 0.016956299841680886, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12728777111111111, "avg_duration_per_task_ms": 101323.44222222222, "avg_output_tokens_per_task": 3885.56, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.7166669999999999, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04331858, "avg_duration_per_task_ms": 119875.46, "avg_output_tokens_per_task": 9185.81, "last_run_timestamp": "2026-10-24T12:22:21.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.015275361427802623, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.058710445, "avg_duration_per_task_ms": 517833.175, "avg_output_tokens_per_task": 13935.145, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.716250125, "accuracy_std_dev": 0.028550118578727003, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2090713325, "avg_duration_per_task_ms": 1406671.1525, "avg_output_tokens_per_task": 13817.695, "last_run_timestamp": "2026-10-24T19:32:17.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7162160675675675, "accuracy_std_dev": 0.02984900053044771, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.1296834662162162, "avg_duration_per_task_ms": 769826.7162162162, "avg_output_tokens_per_task": 28396.02027027027, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7161110833333333, "accuracy_std_dev": 0.032025355021342926, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.04424442, "avg_duration_per_task_ms": 122982.75, "avg_output_tokens_per_task": 8577.505, "last_run_timestamp": "2026-10-24T19:43:58.683Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7155554444444445, "accuracy_std_dev": 0.018324967789034047, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02354926, "avg_duration_per_task_ms": 203685.72222222222, "avg_output_tokens_per_task": 9821.226666666667, "last_run_timestamp": "2026-10-24T19:46:12.078Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7155553333333334, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13937678, "avg_duration_per_task_ms": 263819.3466666667, "avg_output_tokens_per_task": 10311.9, "last_run_timestamp": "2026-10-03T16:18:50.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.07260889583333334, "avg_duration_per_task_ms": 556749.9791666666, "avg_output_tokens_per_task": 12329.4375, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.009582875, "avg_duration_per_task_ms": 164124.5, "avg_output_tokens_per_task": 18445.333333333332, "last_run_timestamp": "2026-10-10T11:44:02.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.715, "accuracy_std_dev": 0.029202378139459823, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09407665500000001, "avg_duration_per_task_ms": 83159.84, "avg_output_tokens_per_task": 3660.855, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7148716923076923, "accuracy_std_dev": 0.0339082212572173, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.03673959538461538, "avg_duration_per_task_ms": 186566.58, "avg_output_tokens_per_task": 9052.430769230768, "last_run_timestamp": "2026-10-24T19:49:36.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7148146666666667, "accuracy_std_dev": 0.031075936024161484, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09082762666666666, "avg_duration_per_task_ms": 156907.88888888888, "avg_output_tokens_per_task": 48272.70222222222, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7144445, "accuracy_std_dev": 0.03331469664302328, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.122428735, "avg_duration_per_task_ms": 164804.72666666665, "avg_output_tokens_per_task": 11653.74, "last_run_timestamp": "2026-10-24T19:52:24.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7133782062780268, "accuracy_std_dev": 0.10832537981105755, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.20968350448430492, "avg_duration_per_task_ms": 1204177.4775784754, "avg_output_tokens_per_task": 56724.81614349776, "last_run_timestamp": "2026-10-10T19:16:51.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7133333846153846, "accuracy_std_dev": 0.03807331209762081, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.08142609230769231, "avg_duration_per_task_ms": 97247.61076923077, "avg_output_tokens_per_task": 8604.487692307692, "last_run_timestamp": "2026-10-24T23:37:48.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7133333749999999, "accuracy_std_dev": 0.059814328458859874, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10449861, "avg_duration_per_task_ms": 230117.0725, "avg_output_tokens_per_task": 11136.0225, "last_run_timestamp": "2026-10-17T19:18:15.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.14919525114109886, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06109567666666668, "avg_duration_per_task_ms": 151262.36, "avg_output_tokens_per_task": 17284.825, "last_run_timestamp": "2026-10-24T18:32:59.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.03953422333736796, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10266909833333333, "avg_duration_per_task_ms": 104721.82833333334, "avg_output_tokens_per_task": 9163.678333333333, "last_run_timestamp": "2026-10-24T19:12:36.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.71333325, "accuracy_std_dev": 0.016996797086142445, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01124519, "avg_duration_per_task_ms": 291598.365, "avg_output_tokens_per_task": 13318.715, "last_run_timestamp": "2026-10-24T13:02:30.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7133332222222223, "accuracy_std_dev": 0.02222223333499722, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06405054444444444, "avg_duration_per_task_ms": 308107.3911111111, "avg_output_tokens_per_task": 9732.942222222222, "last_run_timestamp": "2026-10-24T20:27:01.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05452906, "avg_duration_per_task_ms": 272872.18, "avg_output_tokens_per_task": 39719.44, "last_run_timestamp": "2026-10-13T20:01:29.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.712121101010101, "accuracy_std_dev": 0.029111223892169177, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12218062626262625, "avg_duration_per_task_ms": 537818.2474747475, "avg_output_tokens_per_task": 55422.06060606061, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.7118518888888888, "accuracy_std_dev": 0.032626193960083476, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07140289111111112, "avg_duration_per_task_ms": 302781.4844444444, "avg_output_tokens_per_task": 10769.26888888889, "last_run_timestamp": "2026-10-24T22:01:31.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.711711777027027, "accuracy_std_dev": 0.11123612518334841, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.07281375000000001, "avg_duration_per_task_ms": 155901.73648648648, "avg_output_tokens_per_task": 15030.263513513513, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.71166675, "accuracy_std_dev": 0.019075733568790994, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07509325, "avg_duration_per_task_ms": 322731.32, "avg_output_tokens_per_task": 33483.435, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7113821300813009, "accuracy_std_dev": 0.04920965638205573, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.03321126422764228, "avg_duration_per_task_ms": 1038484.2195121951, "avg_output_tokens_per_task": 47360.833333333336, "last_run_timestamp": "2026-10-17T19:19:46.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.7113332, "accuracy_std_dev": 0.017587783639788142, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.28712958000000005, "avg_duration_per_task_ms": 155529.044, "avg_output_tokens_per_task": 10850.46, "last_run_timestamp": "2026-10-24T19:05:28.397Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7112820769230769, "accuracy_std_dev": 0.0340322401972047, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.04383589384615384, "avg_duration_per_task_ms": 144055.18, "avg_output_tokens_per_task": 8486.646153846154, "last_run_timestamp": "2026-10-24T19:47:13.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.011331263428624743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07186812, "avg_duration_per_task_ms": 348295.26, "avg_output_tokens_per_task": 33945.17333333333, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7108843469387754, "accuracy_std_dev": 0.017917000000000016, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.04385853061224489, "avg_duration_per_task_ms": 61398.17346938775, "avg_output_tokens_per_task": 8040.4795918367345, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7106667999999999, "accuracy_std_dev": 0.037381050458220114, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.144933328, "avg_duration_per_task_ms": 336674.688, "avg_output_tokens_per_task": 10779.192, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7103705555555555, "accuracy_std_dev": 0.030809822424065814, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03036840888888889, "avg_duration_per_task_ms": 105409.86666666667, "avg_output_tokens_per_task": 5076.06, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.7103703333333333, "accuracy_std_dev": 0.03759572984617624, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06715483777777777, "avg_duration_per_task_ms": 69489.32, "avg_output_tokens_per_task": 9660.7, "last_run_timestamp": "2026-10-24T19:30:31.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.71, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01784487, "avg_duration_per_task_ms": 239614.14, "avg_output_tokens_per_task": 16662.97, "last_run_timestamp": "2026-10-24T11:51:42.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12391315, "avg_duration_per_task_ms": 76197.77, "avg_output_tokens_per_task": 4134.025, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.70990758186398, "accuracy_std_dev": 0.05927908384168378, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.04869609571788413, "avg_duration_per_task_ms": 986154.4534005037, "avg_output_tokens_per_task": 53874.76574307305, "last_run_timestamp": "2026-10-24T12:28:11.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7093335999999999, "accuracy_std_dev": 0.027840743923968677, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.049385444, "avg_duration_per_task_ms": 869963.78, "avg_output_tokens_per_task": 8710.828, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.01749773883169545, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11639628666666667, "avg_duration_per_task_ms": 755791.72, "avg_output_tokens_per_task": 27174.273333333334, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7079150969899666, "accuracy_std_dev": 0.016686286941344667, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.04951336454849498, "avg_duration_per_task_ms": 411808.29431438126, "avg_output_tokens_per_task": 8694.879598662208, "last_run_timestamp": "2026-10-24T21:22:48.118Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.7074075555555556, "accuracy_std_dev": 0.05206310721745101, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03110885111111111, "avg_duration_per_task_ms": 310847.1533333333, "avg_output_tokens_per_task": 10019.133333333333, "last_run_timestamp": "2026-10-24T19:49:45.253Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.7074073333333334, "accuracy_std_dev": 0.012350780677080023, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02628152222222222, "avg_duration_per_task_ms": 205434.89555555556, "avg_output_tokens_per_task": 10011.104444444445, "last_run_timestamp": "2026-10-24T19:34:31.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7073334, "accuracy_std_dev": 0.025681872553223187, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024487520000000002, "avg_duration_per_task_ms": 117939.624, "avg_output_tokens_per_task": 30855.352, "last_run_timestamp": "2026-10-24T19:30:58.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7072222499999999, "accuracy_std_dev": 0.02940247900581683, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11202355833333333, "avg_duration_per_task_ms": 94561.55833333333, "avg_output_tokens_per_task": 9737.35, "last_run_timestamp": "2026-10-24T19:14:21.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07705153, "avg_duration_per_task_ms": 159916.36, "avg_output_tokens_per_task": 34851.38, "last_run_timestamp": "2026-10-24T12:16:31.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.06, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02912445, "avg_duration_per_task_ms": 175317.22, "avg_output_tokens_per_task": 10432.57, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.04388527130041595, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06779559555555556, "avg_duration_per_task_ms": 346332.98, "avg_output_tokens_per_task": 9979.706666666667, "last_run_timestamp": "2026-10-24T20:13:56.072Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.7066665000000001, "accuracy_std_dev": 0.027888567949071897, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021449605000000004, "avg_duration_per_task_ms": 108412.625, "avg_output_tokens_per_task": 11247.685, "last_run_timestamp": "2026-10-24T11:28:47.983Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7060302060301508, "accuracy_std_dev": 0.10430171386655158, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.07453727386934673, "avg_duration_per_task_ms": 742520.2713567839, "avg_output_tokens_per_task": 45595.6608040201, "last_run_timestamp": "2026-10-24T18:56:22.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7056452258064517, "accuracy_std_dev": 0.012405393094940622, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.6654760282258064, "avg_duration_per_task_ms": 686538.5927419355, "avg_output_tokens_per_task": 55521.342741935485, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7051358812949641, "accuracy_std_dev": 0.03208361099796414, "run_count": 34, "total_questions": 1668, "avg_cost_per_task_usd": 0.09995068285371703, "avg_duration_per_task_ms": 72045.10491606714, "avg_output_tokens_per_task": 8049.988009592326, "last_run_timestamp": "2026-10-24T19:37:14.677Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.70512827090301, "accuracy_std_dev": 0.04983484902405144, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.1901660066889632, "avg_duration_per_task_ms": 613546.7023411371, "avg_output_tokens_per_task": 9664.80602006689, "last_run_timestamp": "2026-10-24T21:19:54.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7046738967642526, "accuracy_std_dev": 0.03234510691217085, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.07402583821263482, "avg_duration_per_task_ms": 210999.13251155624, "avg_output_tokens_per_task": 9489.39907550077, "last_run_timestamp": "2026-10-24T19:09:54.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.70395825, "accuracy_std_dev": 0.024642497310286955, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.13601082625, "avg_duration_per_task_ms": 82905.08, "avg_output_tokens_per_task": 10495.6875, "last_run_timestamp": "2026-10-24T19:06:47.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.7033335, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01719149, "avg_duration_per_task_ms": 73722.59, "avg_output_tokens_per_task": 12133.38, "last_run_timestamp": "2026-10-24T12:37:04.711Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.703333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.012750089999999999, "avg_duration_per_task_ms": 89612.54, "avg_output_tokens_per_task": 11420.52, "last_run_timestamp": "2026-10-24T12:33:10.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.7022222222222222, "accuracy_std_dev": 0.06584673187668091, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07167732666666665, "avg_duration_per_task_ms": 274025.66444444447, "avg_output_tokens_per_task": 26720.986666666668, "last_run_timestamp": "2026-10-24T18:56:04.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.701764705882353, "accuracy_std_dev": 0.03595790471184932, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.1260328188235294, "avg_duration_per_task_ms": 127118.39411764705, "avg_output_tokens_per_task": 8942.11294117647, "last_run_timestamp": "2026-10-24T19:19:29.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.7016665000000001, "accuracy_std_dev": 0.005000166672222055, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.052099805000000006, "avg_duration_per_task_ms": 224630.845, "avg_output_tokens_per_task": 6764.525, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7012261270903011, "accuracy_std_dev": 0.02520740151666215, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.025383806020066887, "avg_duration_per_task_ms": 238193.83946488294, "avg_output_tokens_per_task": 9583.702341137123, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.701111, "accuracy_std_dev": 0.011811220865487748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13736028333333333, "avg_duration_per_task_ms": 494538.36333333334, "avg_output_tokens_per_task": 57774.223333333335, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.7000836992481203, "accuracy_std_dev": 0.028456037794892915, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.020681045112781956, "avg_duration_per_task_ms": 573578.8195488722, "avg_output_tokens_per_task": 9328.899749373433, "last_run_timestamp": "2026-10-24T19:39:11.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.7000003333333334, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010915493333333333, "avg_duration_per_task_ms": 115952.82, "avg_output_tokens_per_task": 5345.16, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7000003333333332, "accuracy_std_dev": 0.024944382578492925, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12734470666666667, "avg_duration_per_task_ms": 591806.9066666666, "avg_output_tokens_per_task": 57532.753333333334, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07376152, "avg_duration_per_task_ms": 750952.84, "avg_output_tokens_per_task": 11282.28, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01919496, "avg_duration_per_task_ms": 285286.98, "avg_output_tokens_per_task": 14572.04, "last_run_timestamp": "2026-10-24T12:28:14.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.019795359999999998, "avg_duration_per_task_ms": 396615.94, "avg_output_tokens_per_task": 23218.96, "last_run_timestamp": "2026-10-01T08:03:06.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6986667999999999, "accuracy_std_dev": 0.029028574428655626, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04290838, "avg_duration_per_task_ms": 173950.452, "avg_output_tokens_per_task": 8370.488, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.698333375, "accuracy_std_dev": 0.13007476749540772, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0452811775, "avg_duration_per_task_ms": 317194.5925, "avg_output_tokens_per_task": 10517.9925, "last_run_timestamp": "2026-10-24T21:05:35.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.6977779999999999, "accuracy_std_dev": 0.03624346343825322, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10053378, "avg_duration_per_task_ms": 541324.4933333333, "avg_output_tokens_per_task": 9996.04, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6977778333333333, "accuracy_std_dev": 0.017915990515148434, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21024033333333336, "avg_duration_per_task_ms": 335029.48333333334, "avg_output_tokens_per_task": 34166.46666666667, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.6972220833333334, "accuracy_std_dev": 0.04077554452008854, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09417427166666667, "avg_duration_per_task_ms": 206387.00666666665, "avg_output_tokens_per_task": 7840.663333333333, "last_run_timestamp": "2026-10-24T19:36:22.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.6970371111111111, "accuracy_std_dev": 0.052409631153580175, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0251458, "avg_duration_per_task_ms": 210354.41555555555, "avg_output_tokens_per_task": 9795.00888888889, "last_run_timestamp": "2026-10-24T19:49:21.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6969084516129033, "accuracy_std_dev": 0.011597141847886495, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.02167151008064516, "avg_duration_per_task_ms": 544748.2903225806, "avg_output_tokens_per_task": 18890.834677419356, "last_run_timestamp": "2026-10-24T19:03:53.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.696667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11266726, "avg_duration_per_task_ms": 191409.04, "avg_output_tokens_per_task": 8829.08, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14710045000000002, "avg_duration_per_task_ms": 163276.98, "avg_output_tokens_per_task": 5053.91, "last_run_timestamp": "2026-10-01T08:02:59.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6956845625, "accuracy_std_dev": 0.03279928505657131, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.045980859375, "avg_duration_per_task_ms": 547809.8258928572, "avg_output_tokens_per_task": 9085.082589285714, "last_run_timestamp": "2026-10-24T20:03:06.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6952381877551019, "accuracy_std_dev": 0.03106996917024538, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.016887012244897963, "avg_duration_per_task_ms": 257280.0285714286, "avg_output_tokens_per_task": 8914.297959183674, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.6948148888888889, "accuracy_std_dev": 0.030756431516909117, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04968136888888889, "avg_duration_per_task_ms": 351365.8222222222, "avg_output_tokens_per_task": 9136.788888888888, "last_run_timestamp": "2026-10-24T19:50:05.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6937392385786801, "accuracy_std_dev": 0.0037967340817470817, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.005706243654822335, "avg_duration_per_task_ms": 284564.64467005077, "avg_output_tokens_per_task": 8902.248730964468, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01276862, "avg_duration_per_task_ms": 66785.21, "avg_output_tokens_per_task": 8630.75, "last_run_timestamp": "2026-10-24T12:58:47.706Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01353958, "avg_duration_per_task_ms": 117341.27, "avg_output_tokens_per_task": 12550.54, "last_run_timestamp": "2026-10-24T12:06:17.107Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07614706, "avg_duration_per_task_ms": 539148.38, "avg_output_tokens_per_task": 13067.92, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03237594, "avg_duration_per_task_ms": 69982.76, "avg_output_tokens_per_task": 15576.66, "last_run_timestamp": "2026-10-20T06:20:53.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02754302, "avg_duration_per_task_ms": 209408.8, "avg_output_tokens_per_task": 14914.54, "last_run_timestamp": "2026-10-24T12:03:24.771Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01748764, "avg_duration_per_task_ms": 289873.3, "avg_output_tokens_per_task": 9630.44, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6930301363636364, "accuracy_std_dev": 0.023993062780151963, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.10643902272727274, "avg_duration_per_task_ms": 76241.01909090909, "avg_output_tokens_per_task": 9720.23090909091, "last_run_timestamp": "2026-10-24T19:25:31.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6917516734693877, "accuracy_std_dev": 0.023775708138770536, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.03340652295918367, "avg_duration_per_task_ms": 558953.0178571428, "avg_output_tokens_per_task": 18907.030612244896, "last_run_timestamp": "2026-10-17T19:21:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.6905556666666667, "accuracy_std_dev": 0.0689850851662074, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11194472833333334, "avg_duration_per_task_ms": 74478.39833333333, "avg_output_tokens_per_task": 9151.01, "last_run_timestamp": "2026-10-24T19:20:41.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.690093718875502, "accuracy_std_dev": 0.022147823156238193, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.006209787148594378, "avg_duration_per_task_ms": 270789.8915662651, "avg_output_tokens_per_task": 9428.562248995984, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6891892297297297, "accuracy_std_dev": 0.03170486152346707, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6622049662162162, "avg_duration_per_task_ms": 777791.8783783783, "avg_output_tokens_per_task": 59355.4527027027, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6888886666666667, "accuracy_std_dev": 0.04262959037257258, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01831292444444444, "avg_duration_per_task_ms": 377813.63333333336, "avg_output_tokens_per_task": 20424.906666666666, "last_run_timestamp": "2026-10-24T21:02:58.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.6886632385786803, "accuracy_std_dev": 0.0900995905873051, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.1329306802030457, "avg_duration_per_task_ms": 1780497.6497461929, "avg_output_tokens_per_task": 98848.87817258883, "last_run_timestamp": "2026-10-24T19:18:22.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01622654, "avg_duration_per_task_ms": 255665.46, "avg_output_tokens_per_task": 10102.66, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6866668, "accuracy_std_dev": 0.03373094853335731, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.035621146, "avg_duration_per_task_ms": 161374.6, "avg_output_tokens_per_task": 10041.704, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.68666675, "accuracy_std_dev": 0.14596428025783398, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.093884975, "avg_duration_per_task_ms": 1131069.095, "avg_output_tokens_per_task": 24772.465, "last_run_timestamp": "2026-10-17T19:19:25.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01561806, "avg_duration_per_task_ms": 117250.14, "avg_output_tokens_per_task": 14519.81, "last_run_timestamp": "2026-10-24T12:06:50.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.6863144308943089, "accuracy_std_dev": 0.02557346986546797, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.023131280487804877, "avg_duration_per_task_ms": 301545.0325203252, "avg_output_tokens_per_task": 16644.90243902439, "last_run_timestamp": "2026-10-24T20:27:13.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6837050185873607, "accuracy_std_dev": 0.11487260701179351, "run_count": 11, "total_questions": 538, "avg_cost_per_task_usd": 0.32785563568773235, "avg_duration_per_task_ms": 1690067.0799256505, "avg_output_tokens_per_task": 54783.62267657993, "last_run_timestamp": "2026-10-24T12:39:42.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6833335555555555, "accuracy_std_dev": 0.09394000318064849, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05554264666666667, "avg_duration_per_task_ms": 504546.8688888889, "avg_output_tokens_per_task": 15954.128888888888, "last_run_timestamp": "2026-10-24T19:51:41.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.07779388246014092, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00909372, "avg_duration_per_task_ms": 253755.2, "avg_output_tokens_per_task": 15941.146666666667, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.683333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00634196, "avg_duration_per_task_ms": 404260.22, "avg_output_tokens_per_task": 17374, "last_run_timestamp": "2026-10-17T11:31:13.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6822220000000001, "accuracy_std_dev": 0.020607975203789406, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.015206260000000001, "avg_duration_per_task_ms": 252877.44, "avg_output_tokens_per_task": 17489.156666666666, "last_run_timestamp": "2026-10-24T20:33:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.682, "accuracy_std_dev": 0.03438995977607416, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.017993061999999997, "avg_duration_per_task_ms": 315808.552, "avg_output_tokens_per_task": 20586.058, "last_run_timestamp": "2026-10-10T21:10:48.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6816665000000001, "accuracy_std_dev": 0.02327362599059285, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013877145, "avg_duration_per_task_ms": 70751.9, "avg_output_tokens_per_task": 5304.49, "last_run_timestamp": "2026-10-01T19:04:52.652Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.6811112222222222, "accuracy_std_dev": 0.11108899779983991, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09013350666666665, "avg_duration_per_task_ms": 216735.6, "avg_output_tokens_per_task": 20823.96222222222, "last_run_timestamp": "2026-10-24T19:19:49.639Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01807364, "avg_duration_per_task_ms": 208774.72, "avg_output_tokens_per_task": 16251.38, "last_run_timestamp": "2026-10-24T12:17:54.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01975412, "avg_duration_per_task_ms": 454730.8, "avg_output_tokens_per_task": 9931.28, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.6785714285714286, "accuracy_std_dev": 0.020303791411297286, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.017469171428571427, "avg_duration_per_task_ms": 119261.06857142856, "avg_output_tokens_per_task": 21365.42, "last_run_timestamp": "2026-10-24T11:42:22.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6773334, "accuracy_std_dev": 0.025854265068649684, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.00809352, "avg_duration_per_task_ms": 287886.284, "avg_output_tokens_per_task": 12626.116, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.676667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01545023, "avg_duration_per_task_ms": 128578.81, "avg_output_tokens_per_task": 15653.52, "last_run_timestamp": "2026-10-24T12:17:01.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.676, "accuracy_std_dev": 0.19009238689542513, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011841847999999999, "avg_duration_per_task_ms": 391925.292, "avg_output_tokens_per_task": 10879.944, "last_run_timestamp": "2026-10-24T13:02:10.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6755553333333334, "accuracy_std_dev": 0.01422909556351193, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.015154949999999999, "avg_duration_per_task_ms": 230024.84, "avg_output_tokens_per_task": 10278.443333333333, "last_run_timestamp": "2026-10-13T22:01:15.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6746668, "accuracy_std_dev": 0.022070706580442783, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.00831342, "avg_duration_per_task_ms": 264349.076, "avg_output_tokens_per_task": 9566.64, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6738888333333334, "accuracy_std_dev": 0.07846715716233696, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.051633876666666675, "avg_duration_per_task_ms": 318522.69, "avg_output_tokens_per_task": 7956.61, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64995528, "avg_duration_per_task_ms": 85231.23, "avg_output_tokens_per_task": 3880.06, "last_run_timestamp": "2026-10-01T08:05:26.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.02372680934208296, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11501549333333332, "avg_duration_per_task_ms": 145439.36666666667, "avg_output_tokens_per_task": 56059.84, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09564197999999999, "avg_duration_per_task_ms": 166878.82, "avg_output_tokens_per_task": 4054.64, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.04504326, "avg_duration_per_task_ms": 481727.7, "avg_output_tokens_per_task": 10202.42, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.6725926666666666, "accuracy_std_dev": 0.08293649247064486, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09095060666666666, "avg_duration_per_task_ms": 106719.20666666667, "avg_output_tokens_per_task": 8550.382222222222, "last_run_timestamp": "2026-10-24T19:23:31.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6706667, "accuracy_std_dev": 0.0663460470232402, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.061452046, "avg_duration_per_task_ms": 791003.158, "avg_output_tokens_per_task": 18308.462, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6689523428571428, "accuracy_std_dev": 0.030318660008971234, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.12536032800000002, "avg_duration_per_task_ms": 121567.65485714286, "avg_output_tokens_per_task": 12712.501714285714, "last_run_timestamp": "2026-10-24T13:04:43.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.668691016194332, "accuracy_std_dev": 0.033329650538972036, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.021791346153846154, "avg_duration_per_task_ms": 276569.14979757083, "avg_output_tokens_per_task": 16852.0971659919, "last_run_timestamp": "2026-10-24T19:52:46.621Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.666111, "accuracy_std_dev": 0.03259831473404721, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.117344875, "avg_duration_per_task_ms": 125188.30166666667, "avg_output_tokens_per_task": 12661.381666666666, "last_run_timestamp": "2026-10-24T12:09:26.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6649998333333333, "accuracy_std_dev": 0.024626931209664655, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008708743333333335, "avg_duration_per_task_ms": 613076.7966666666, "avg_output_tokens_per_task": 17440.79, "last_run_timestamp": "2026-10-24T19:39:37.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6640001999999999, "accuracy_std_dev": 0.027519806201352526, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023235323999999998, "avg_duration_per_task_ms": 326459.196, "avg_output_tokens_per_task": 10364.68, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.6632998585858586, "accuracy_std_dev": 0.0034015000000000017, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.07516567676767677, "avg_duration_per_task_ms": 78894.82828282828, "avg_output_tokens_per_task": 13736.282828282829, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6625000833333333, "accuracy_std_dev": 0.026566641399376315, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11466395333333333, "avg_duration_per_task_ms": 126856.15333333334, "avg_output_tokens_per_task": 12309.436666666666, "last_run_timestamp": "2026-10-24T12:22:57.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.66, "accuracy_std_dev": 0.017384335224563518, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.026960704, "avg_duration_per_task_ms": 420608.976, "avg_output_tokens_per_task": 10298.344, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.013329200000000001, "avg_duration_per_task_ms": 278453.68, "avg_output_tokens_per_task": 15070.16, "last_run_timestamp": "2026-10-24T12:29:49.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.66, "accuracy_std_dev": 0.1609540552497306, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1285084711111111, "avg_duration_per_task_ms": 355132.86666666664, "avg_output_tokens_per_task": 8634.402222222223, "last_run_timestamp": "2026-10-24T19:43:33.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.659259222222222, "accuracy_std_dev": 0.028664879599443933, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.12198797388888888, "avg_duration_per_task_ms": 121356.62555555555, "avg_output_tokens_per_task": 12615.958333333334, "last_run_timestamp": "2026-10-24T12:56:50.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.6588886666666667, "accuracy_std_dev": 0.03965918385723817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006265106666666667, "avg_duration_per_task_ms": 362004.0733333333, "avg_output_tokens_per_task": 19036.246666666666, "last_run_timestamp": "2026-10-24T19:34:04.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6585712857142858, "accuracy_std_dev": 0.014125931834894325, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.023240128571428573, "avg_duration_per_task_ms": 158768.29142857142, "avg_output_tokens_per_task": 17172.022857142856, "last_run_timestamp": "2026-10-24T19:17:27.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.6573334, "accuracy_std_dev": 0.0067987188381341416, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.029082692, "avg_duration_per_task_ms": 234875.74, "avg_output_tokens_per_task": 10738.14, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6566381031518624, "accuracy_std_dev": 0.020222279406836508, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.01698823782234957, "avg_duration_per_task_ms": 325791.0286532951, "avg_output_tokens_per_task": 22916.045845272205, "last_run_timestamp": "2026-10-24T12:47:29.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.656, "accuracy_std_dev": 0.02480152325160696, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022391364000000004, "avg_duration_per_task_ms": 255228.56, "avg_output_tokens_per_task": 9636.224, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6557823102040816, "accuracy_std_dev": 0.009318408855593308, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.014797057142857142, "avg_duration_per_task_ms": 277907.2775510204, "avg_output_tokens_per_task": 20245.579591836733, "last_run_timestamp": "2026-10-10T11:18:54.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.654618485943775, "accuracy_std_dev": 0.11865847802618236, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.04366364257028112, "avg_duration_per_task_ms": 1933703.5220883535, "avg_output_tokens_per_task": 71251.46586345382, "last_run_timestamp": "2026-10-24T12:24:29.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.65416675, "accuracy_std_dev": 0.03373725891262509, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.013133230000000001, "avg_duration_per_task_ms": 415006.955, "avg_output_tokens_per_task": 18075.7325, "last_run_timestamp": "2026-10-24T19:51:37.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.023726809342082957, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03210012, "avg_duration_per_task_ms": 143919.89333333334, "avg_output_tokens_per_task": 17353.273333333334, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.653333142857143, "accuracy_std_dev": 0.10498677770139653, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.017404814285714287, "avg_duration_per_task_ms": 257125.88, "avg_output_tokens_per_task": 9629.577142857142, "last_run_timestamp": "2026-10-24T19:35:00.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.023979720000000003, "avg_duration_per_task_ms": 166174, "avg_output_tokens_per_task": 10916.26, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07713578, "avg_duration_per_task_ms": 238048.66, "avg_output_tokens_per_task": 10413.86, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.653243932885906, "accuracy_std_dev": 0.02323319256351424, "run_count": 15, "total_questions": 745, "avg_cost_per_task_usd": 0.010499983892617451, "avg_duration_per_task_ms": 309969.21610738256, "avg_output_tokens_per_task": 17108.903355704697, "last_run_timestamp": "2026-10-24T18:50:11.994Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.65166675, "accuracy_std_dev": 0.038657908491504044, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021260025000000002, "avg_duration_per_task_ms": 249515.925, "avg_output_tokens_per_task": 26907.185, "last_run_timestamp": "2026-10-24T12:38:08.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.65166675, "accuracy_std_dev": 0.012801703448662586, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.031589305, "avg_duration_per_task_ms": 139715.76, "avg_output_tokens_per_task": 17049.6, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.650067849593496, "accuracy_std_dev": 0.11323840378051966, "run_count": 10, "total_questions": 492, "avg_cost_per_task_usd": 0.025260294715447155, "avg_duration_per_task_ms": 965913.6768292683, "avg_output_tokens_per_task": 32745.57113821138, "last_run_timestamp": "2026-10-17T21:28:32.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6500000833333334, "accuracy_std_dev": 0.02441931916010468, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11641354333333334, "avg_duration_per_task_ms": 142724.34, "avg_output_tokens_per_task": 12635.95, "last_run_timestamp": "2026-10-24T11:52:54.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.65, "accuracy_std_dev": 0.025963164791681315, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.022032366666666667, "avg_duration_per_task_ms": 277950.80666666664, "avg_output_tokens_per_task": 10197.613333333333, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.6499997499999999, "accuracy_std_dev": 0.0378593669609979, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015033419999999999, "avg_duration_per_task_ms": 89205.15, "avg_output_tokens_per_task": 19468.825, "last_run_timestamp": "2026-10-24T19:10:30.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6490788090452262, "accuracy_std_dev": 0.06101351971284725, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.02417512814070352, "avg_duration_per_task_ms": 427158.12060301506, "avg_output_tokens_per_task": 15670.065326633166, "last_run_timestamp": "2026-10-24T19:06:03.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6477778888888889, "accuracy_std_dev": 0.032735479299800314, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.033562786666666664, "avg_duration_per_task_ms": 67754.29111111112, "avg_output_tokens_per_task": 16126.108888888888, "last_run_timestamp": "2026-10-24T18:54:16.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6477778333333334, "accuracy_std_dev": 0.03430091421724631, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09916395333333333, "avg_duration_per_task_ms": 153090.92333333334, "avg_output_tokens_per_task": 12859.136666666667, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6475644699140402, "accuracy_std_dev": 0.027096557835463376, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.014083598853868194, "avg_duration_per_task_ms": 819841.329512894, "avg_output_tokens_per_task": 25954.131805157594, "last_run_timestamp": "2026-10-24T20:01:47.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6460437828282828, "accuracy_std_dev": 0.038169636351874946, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.012887035353535355, "avg_duration_per_task_ms": 941376.398989899, "avg_output_tokens_per_task": 22837.7297979798, "last_run_timestamp": "2026-10-24T14:58:16.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.6437036666666667, "accuracy_std_dev": 0.12512151876564725, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05068651111111112, "avg_duration_per_task_ms": 788989.3488888888, "avg_output_tokens_per_task": 25827.417777777777, "last_run_timestamp": "2026-10-24T19:37:12.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6433335, "accuracy_std_dev": 0.014272415770172448, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031060726666666667, "avg_duration_per_task_ms": 229987.72666666665, "avg_output_tokens_per_task": 16358.803333333333, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6415203403508771, "accuracy_std_dev": 0.08180204086577275, "run_count": 6, "total_questions": 285, "avg_cost_per_task_usd": 0.12277472280701755, "avg_duration_per_task_ms": 351597.06315789477, "avg_output_tokens_per_task": 9359.947368421053, "last_run_timestamp": "2026-10-17T19:51:53.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.6403186580882353, "accuracy_std_dev": 0.0713480377599717, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.024952294117647062, "avg_duration_per_task_ms": 269151.14705882355, "avg_output_tokens_per_task": 25044.069852941175, "last_run_timestamp": "2026-10-17T19:25:29.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.64, "accuracy_std_dev": 0.02022095722758942, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024095932, "avg_duration_per_task_ms": 145745.204, "avg_output_tokens_per_task": 10183.88, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.64, "accuracy_std_dev": 0.03238665020035263, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021093272000000003, "avg_duration_per_task_ms": 219291.42, "avg_output_tokens_per_task": 9989.872, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6386381963882618, "accuracy_std_dev": 0.04009666605876078, "run_count": 18, "total_questions": 886, "avg_cost_per_task_usd": 0.0201472381489842, "avg_duration_per_task_ms": 321295.43115124153, "avg_output_tokens_per_task": 17837.213318284423, "last_run_timestamp": "2026-10-24T19:19:43.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.04228062043326973, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08944130666666666, "avg_duration_per_task_ms": 118157.82666666666, "avg_output_tokens_per_task": 12612.513333333334, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.032810732962790655, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.20948366666666668, "avg_duration_per_task_ms": 978205.8133333334, "avg_output_tokens_per_task": 27155.76, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.6366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.008583809999999999, "avg_duration_per_task_ms": 1454771.53, "avg_output_tokens_per_task": 17935.13, "last_run_timestamp": "2026-10-24T20:43:26.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6353466979865772, "accuracy_std_dev": 0.036572065936844325, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.033807637583892616, "avg_duration_per_task_ms": 262695.20805369125, "avg_output_tokens_per_task": 17180.040268456374, "last_run_timestamp": "2026-10-24T18:33:47.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6307017947368421, "accuracy_std_dev": 0.04099517109977711, "run_count": 4, "total_questions": 190, "avg_cost_per_task_usd": 0.02501245263157895, "avg_duration_per_task_ms": 66232.84736842105, "avg_output_tokens_per_task": 14146.842105263158, "last_run_timestamp": "2026-10-24T18:40:18.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6293332, "accuracy_std_dev": 0.02133341250151978, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.028055644, "avg_duration_per_task_ms": 202913.916, "avg_output_tokens_per_task": 9696.648, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6284643303370787, "accuracy_std_dev": 0.02768998520123195, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.041866957303370776, "avg_duration_per_task_ms": 82960.69662921349, "avg_output_tokens_per_task": 22261.50786516854, "last_run_timestamp": "2026-10-24T19:23:06.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12269806, "avg_duration_per_task_ms": 610304.8666666667, "avg_output_tokens_per_task": 27046.16, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.622962888888889, "accuracy_std_dev": 0.02416214886251472, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06782037777777777, "avg_duration_per_task_ms": 31573.64888888889, "avg_output_tokens_per_task": 21383.52666666667, "last_run_timestamp": "2026-10-24T18:29:40.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6216667499999999, "accuracy_std_dev": 0.04997217837144482, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08547625, "avg_duration_per_task_ms": 106037.93, "avg_output_tokens_per_task": 11996.815, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6192592222222222, "accuracy_std_dev": 0.03841860903196025, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09472958, "avg_duration_per_task_ms": 55630.27777777778, "avg_output_tokens_per_task": 3606.46, "last_run_timestamp": "2026-10-24T19:00:38.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.619048, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.019175836734693877, "avg_duration_per_task_ms": 171955.81632653062, "avg_output_tokens_per_task": 9348.57142857143, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.616161808080808, "accuracy_std_dev": 0.009872706603940993, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.12580894444444446, "avg_duration_per_task_ms": 627518.9242424242, "avg_output_tokens_per_task": 29626.045454545456, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.613333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025167079999999998, "avg_duration_per_task_ms": 171182.22, "avg_output_tokens_per_task": 11292.18, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6076188571428572, "accuracy_std_dev": 0.03274303997331158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.046068719999999994, "avg_duration_per_task_ms": 191059.44, "avg_output_tokens_per_task": 3405.154285714286, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6060269955357143, "accuracy_std_dev": 0.032348691330028045, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.19772456473214287, "avg_duration_per_task_ms": 188670.52455357142, "avg_output_tokens_per_task": 13047.183035714286, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6024903390804598, "accuracy_std_dev": 0.08896202865084331, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.031245040229885054, "avg_duration_per_task_ms": 1940985.2241379311, "avg_output_tokens_per_task": 61478.612068965514, "last_run_timestamp": "2026-10-24T19:11:22.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.597222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.09316766666666666, "avg_duration_per_task_ms": 117445.22916666667, "avg_output_tokens_per_task": 3558.4166666666665, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.5966665714285715, "accuracy_std_dev": 0.02742942459089792, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04750767714285715, "avg_duration_per_task_ms": 92834.19142857143, "avg_output_tokens_per_task": 4625.282857142857, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5950784093959732, "accuracy_std_dev": 0.022941326296058443, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.019630268456375836, "avg_duration_per_task_ms": 99814.84899328859, "avg_output_tokens_per_task": 4258.728187919463, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.5942761515151515, "accuracy_std_dev": 0.012018411802313968, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.15294240909090911, "avg_duration_per_task_ms": 508469.1313131313, "avg_output_tokens_per_task": 26213.378787878788, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.5933333333333334, "accuracy_std_dev": 0.02880349773127486, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21649743333333332, "avg_duration_per_task_ms": 191120.76666666666, "avg_output_tokens_per_task": 14122.313333333334, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5881481111111111, "accuracy_std_dev": 0.007554077566224961, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029889999999999996, "avg_duration_per_task_ms": 78610.16, "avg_output_tokens_per_task": 8108.122222222222, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.584762, "accuracy_std_dev": 0.13245977235587955, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03989074857142857, "avg_duration_per_task_ms": 436437.8257142857, "avg_output_tokens_per_task": 8373.411428571428, "last_run_timestamp": "2026-10-10T19:10:33.654Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.5801346121212121, "accuracy_std_dev": 0.2505881880506142, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.015714664646464645, "avg_duration_per_task_ms": 289180.65252525255, "avg_output_tokens_per_task": 8241.527272727273, "last_run_timestamp": "2026-10-17T11:22:12.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5789978786127168, "accuracy_std_dev": 0.05180400838836742, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.18764681502890174, "avg_duration_per_task_ms": 2044873.4393063583, "avg_output_tokens_per_task": 41085.294797687864, "last_run_timestamp": "2026-10-17T19:42:10.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.57483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.017800938775510203, "avg_duration_per_task_ms": 142107.1224489796, "avg_output_tokens_per_task": 3993.1632653061224, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.571111, "accuracy_std_dev": 0.008314764498569157, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028662713333333336, "avg_duration_per_task_ms": 78035.85333333333, "avg_output_tokens_per_task": 8001.593333333333, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.5674075555555556, "accuracy_std_dev": 0.07922128546898255, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03296292666666667, "avg_duration_per_task_ms": 389055.2444444444, "avg_output_tokens_per_task": 17353.69111111111, "last_run_timestamp": "2026-10-24T19:54:36.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5670103642611685, "accuracy_std_dev": 0.038907173466327376, "run_count": 6, "total_questions": 291, "avg_cost_per_task_usd": 0.07826669759450172, "avg_duration_per_task_ms": 96488.71821305843, "avg_output_tokens_per_task": 6239.993127147766, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.5666665, "accuracy_std_dev": 0.1466665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02785182, "avg_duration_per_task_ms": 139069.32, "avg_output_tokens_per_task": 11612.72, "last_run_timestamp": "2026-10-24T12:08:00.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.5622223333333333, "accuracy_std_dev": 0.03456997912768952, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.23936231333333335, "avg_duration_per_task_ms": 165980.19333333333, "avg_output_tokens_per_task": 13303.673333333334, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.5544443333333333, "accuracy_std_dev": 0.10997765317352229, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.040382676666666666, "avg_duration_per_task_ms": 99197.56, "avg_output_tokens_per_task": 4360.85, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5534669598997495, "accuracy_std_dev": 0.05099077839163053, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.0106168671679198, "avg_duration_per_task_ms": 216000.37593984962, "avg_output_tokens_per_task": 17010.057644110275, "last_run_timestamp": "2026-10-24T19:30:50.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5511113333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.035753513333333334, "avg_duration_per_task_ms": 61316.40666666667, "avg_output_tokens_per_task": 6659.566666666667, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5370368653198654, "accuracy_std_dev": 0.03035419429923245, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.0026686632996632998, "avg_duration_per_task_ms": 405513.9057239057, "avg_output_tokens_per_task": 6736.090909090909, "last_run_timestamp": "2026-10-24T19:31:02.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.535956005076142, "accuracy_std_dev": 0.0166977509787995, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.018755781725888324, "avg_duration_per_task_ms": 1279767.0253807106, "avg_output_tokens_per_task": 84685.69796954315, "last_run_timestamp": "2026-10-24T19:37:33.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5333335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10219375, "avg_duration_per_task_ms": 375443.12, "avg_output_tokens_per_task": 17495.06, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.5333333333333333, "accuracy_std_dev": 0.08960819241688912, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05793356666666665, "avg_duration_per_task_ms": 732402.9933333333, "avg_output_tokens_per_task": 42843.026666666665, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.526785875, "accuracy_std_dev": 0.01933119676763796, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.06628725223214285, "avg_duration_per_task_ms": 287109.67633928574, "avg_output_tokens_per_task": 4037.7633928571427, "last_run_timestamp": "2026-10-24T18:33:38.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.5226130251256281, "accuracy_std_dev": 0.08810932391131698, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.05013379899497488, "avg_duration_per_task_ms": 411095.6934673367, "avg_output_tokens_per_task": 7609.560301507538, "last_run_timestamp": "2026-10-24T18:46:40.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5221089591836734, "accuracy_std_dev": 0.023889008586586408, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.13644156632653062, "avg_duration_per_task_ms": 73125.92346938775, "avg_output_tokens_per_task": 2553.7397959183672, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517500125, "accuracy_std_dev": 0.024930426803594342, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0282700725, "avg_duration_per_task_ms": 444815, "avg_output_tokens_per_task": 58343.125, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.5173608125000001, "accuracy_std_dev": 0.025072499999999998, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.06539028125, "avg_duration_per_task_ms": 2881049.1770833335, "avg_output_tokens_per_task": 39867.583333333336, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.5159258888888889, "accuracy_std_dev": 0.032271244532157746, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.030723802222222218, "avg_duration_per_task_ms": 255044.73555555556, "avg_output_tokens_per_task": 20795.11111111111, "last_run_timestamp": "2026-10-24T19:23:13.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5156599127516779, "accuracy_std_dev": 0.03449671802746843, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.024214489932885906, "avg_duration_per_task_ms": 190286.47651006712, "avg_output_tokens_per_task": 48937.10738255033, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.5104164999999999, "accuracy_std_dev": 0.003472500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.024820281250000003, "avg_duration_per_task_ms": 480070.4583333333, "avg_output_tokens_per_task": 9578.8125, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5083332500000001, "accuracy_std_dev": 0.026404802805313633, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12635733, "avg_duration_per_task_ms": 53878.245, "avg_output_tokens_per_task": 2800.47, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.507575826446281, "accuracy_std_dev": 0.10203663931078877, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.05456804958677686, "avg_duration_per_task_ms": 374769.78925619833, "avg_output_tokens_per_task": 24046.93388429752, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5051021836734694, "accuracy_std_dev": 0.011159046957513885, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.019562938775510203, "avg_duration_per_task_ms": 1197227.9285714286, "avg_output_tokens_per_task": 100212.39285714286, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5033670303030303, "accuracy_std_dev": 0.030340499999999992, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.01769851515151515, "avg_duration_per_task_ms": 135563.53535353535, "avg_output_tokens_per_task": 30988.575757575756, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.5033669747474748, "accuracy_std_dev": 0.027292446564651823, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.025276146464646466, "avg_duration_per_task_ms": 577739.3712121212, "avg_output_tokens_per_task": 53598.469696969696, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.4995211379310344, "accuracy_std_dev": 0.03521937973703148, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.16297116666666667, "avg_duration_per_task_ms": 2016445.1580459771, "avg_output_tokens_per_task": 38482.971264367814, "last_run_timestamp": "2026-10-17T19:35:49.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48850570114942526, "accuracy_std_dev": 0.017439968302957548, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.031982068965517245, "avg_duration_per_task_ms": 642915.867816092, "avg_output_tokens_per_task": 59401.63505747126, "last_run_timestamp": "2026-10-23T09:26:25.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.4852150080645161, "accuracy_std_dev": 0.023420064714684302, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.028619552419354842, "avg_duration_per_task_ms": 386610.2439516129, "avg_output_tokens_per_task": 66355.3125, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.4850950081300813, "accuracy_std_dev": 0.026197876665104, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.02379429674796748, "avg_duration_per_task_ms": 820057.8048780488, "avg_output_tokens_per_task": 22035.906504065042, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.48266679999999995, "accuracy_std_dev": 0.013727319074021705, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.067269828, "avg_duration_per_task_ms": 279626.944, "avg_output_tokens_per_task": 4093.296, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47874697986577186, "accuracy_std_dev": 0.00975008705374242, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.16965920805369125, "avg_duration_per_task_ms": 2424316.9261744968, "avg_output_tokens_per_task": 45379.409395973154, "last_run_timestamp": "2026-10-17T19:38:30.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.4777776666666667, "accuracy_std_dev": 0.013698571612479242, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12510532, "avg_duration_per_task_ms": 53067.566666666666, "avg_output_tokens_per_task": 2709.2933333333335, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.4766666666666667, "accuracy_std_dev": 0.029501952227305586, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07812271, "avg_duration_per_task_ms": 102263.82666666666, "avg_output_tokens_per_task": 2763.403333333333, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4766666666666667, "accuracy_std_dev": 0.02301379618595961, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025132050000000003, "avg_duration_per_task_ms": 126606.66333333333, "avg_output_tokens_per_task": 3730.5233333333335, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.4755555, "accuracy_std_dev": 0.023306820916847493, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03474162, "avg_duration_per_task_ms": 44831.98333333333, "avg_output_tokens_per_task": 3658.7366666666667, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.24038894, "avg_duration_per_task_ms": 93856.16, "avg_output_tokens_per_task": 2866.86, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.47222181249999995, "accuracy_std_dev": 0.0011595000000000077, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.07886530208333332, "avg_duration_per_task_ms": 55709.895833333336, "avg_output_tokens_per_task": 3493.7291666666665, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.46820784971098267, "accuracy_std_dev": 0.04097451831844482, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.027368598265895955, "avg_duration_per_task_ms": 2298741.832369942, "avg_output_tokens_per_task": 136175.1647398844, "last_run_timestamp": "2026-10-24T19:05:04.624Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.46777799999999997, "accuracy_std_dev": 0.0387617763868823, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02465040333333333, "avg_duration_per_task_ms": 59075.62, "avg_output_tokens_per_task": 3762.943333333333, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.466667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03175828, "avg_duration_per_task_ms": 435844.14, "avg_output_tokens_per_task": 22556.3, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.4662164729729729, "accuracy_std_dev": 0.03892660093903227, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04219703378378378, "avg_duration_per_task_ms": 711257.6756756756, "avg_output_tokens_per_task": 98782.22297297297, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.4653334, "accuracy_std_dev": 0.021249757124259083, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.020888280000000002, "avg_duration_per_task_ms": 647002.876, "avg_output_tokens_per_task": 20107.14, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.4650001666666666, "accuracy_std_dev": 0.03531613575226233, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.018810040000000004, "avg_duration_per_task_ms": 93827.55333333333, "avg_output_tokens_per_task": 3599.63, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.46166650000000004, "accuracy_std_dev": 0.008333499999999994, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.035224570000000004, "avg_duration_per_task_ms": 486853.18, "avg_output_tokens_per_task": 23253.82, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4615010526315789, "accuracy_std_dev": 0.036598855562569095, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.05408880409356725, "avg_duration_per_task_ms": 193973.8625730994, "avg_output_tokens_per_task": 87403.31578947368, "last_run_timestamp": "2026-10-17T18:58:49.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4608501677852349, "accuracy_std_dev": 0.034994868451198195, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04671855033557047, "avg_duration_per_task_ms": 767600.1241610738, "avg_output_tokens_per_task": 101728.0033557047, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.46068351282051284, "accuracy_std_dev": 0.04225414673955088, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.01900871794871795, "avg_duration_per_task_ms": 170098.1794871795, "avg_output_tokens_per_task": 35854.89743589744, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4600001428571428, "accuracy_std_dev": 0.034226236108027545, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02152209714285714, "avg_duration_per_task_ms": 223527.5657142857, "avg_output_tokens_per_task": 3520.9742857142855, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.454583375, "accuracy_std_dev": 0.040686489790646416, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05330521999999999, "avg_duration_per_task_ms": 171560.5725, "avg_output_tokens_per_task": 3741.72, "last_run_timestamp": "2026-10-17T19:11:29.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4521073362068965, "accuracy_std_dev": 0.045992926219636195, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.015361428160919543, "avg_duration_per_task_ms": 128943.27873563218, "avg_output_tokens_per_task": 3310.485632183908, "last_run_timestamp": "2026-10-24T19:23:50.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4511785656565656, "accuracy_std_dev": 0.03515039216708685, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04521708585858586, "avg_duration_per_task_ms": 779859.6717171717, "avg_output_tokens_per_task": 105433.07575757576, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.44952385714285714, "accuracy_std_dev": 0.047255747470329915, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.053731311428571425, "avg_duration_per_task_ms": 127204.9, "avg_output_tokens_per_task": 3489.102857142857, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.44932439189189194, "accuracy_std_dev": 0.04806295077869809, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.05292330743243243, "avg_duration_per_task_ms": 75551.64864864865, "avg_output_tokens_per_task": 3306.3682432432433, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.4446664000000001, "accuracy_std_dev": 0.027455050942221906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021282972, "avg_duration_per_task_ms": 631261.664, "avg_output_tokens_per_task": 20624.588, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.443749875, "accuracy_std_dev": 0.029881164520637004, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.040143645, "avg_duration_per_task_ms": 278419.015, "avg_output_tokens_per_task": 3569.46, "last_run_timestamp": "2026-10-24T19:41:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.44333349999999994, "accuracy_std_dev": 0.07371123086466087, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.18429278333333332, "avg_duration_per_task_ms": 118282.36333333333, "avg_output_tokens_per_task": 3443.1833333333334, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.44333325, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025700495, "avg_duration_per_task_ms": 61023.305, "avg_output_tokens_per_task": 2441.375, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4416665, "accuracy_std_dev": 0.017873086533948188, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.104832345, "avg_duration_per_task_ms": 890298.59, "avg_output_tokens_per_task": 21334.155, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.4391890202702703, "accuracy_std_dev": 0.03559610686773047, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.054588496621621616, "avg_duration_per_task_ms": 244762.44932432432, "avg_output_tokens_per_task": 2182.847972972973, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.43777799999999994, "accuracy_std_dev": 0.03861798340151904, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.19450383999999998, "avg_duration_per_task_ms": 50798.87333333334, "avg_output_tokens_per_task": 2047.5333333333333, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.011330936570096718, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02595394666666667, "avg_duration_per_task_ms": 66273.3, "avg_output_tokens_per_task": 2407.2933333333335, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.21387673999999998, "avg_duration_per_task_ms": 47267.2, "avg_output_tokens_per_task": 2224.56, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.435, "accuracy_std_dev": 0.020749805890658335, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.074401825, "avg_duration_per_task_ms": 778012.545, "avg_output_tokens_per_task": 20313.29, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.43434374747474747, "accuracy_std_dev": 0.012448999999999988, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.023558444444444444, "avg_duration_per_task_ms": 1871236.9090909092, "avg_output_tokens_per_task": 14653.222222222223, "last_run_timestamp": "2026-10-13T10:40:58.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.43434354545454545, "accuracy_std_dev": 0.022037645087440712, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.06906736868686869, "avg_duration_per_task_ms": 323353.22222222225, "avg_output_tokens_per_task": 3378.3585858585857, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43416662500000003, "accuracy_std_dev": 0.01613729505630899, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2170375225, "avg_duration_per_task_ms": 46329.7025, "avg_output_tokens_per_task": 2265.4175, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4333331666666667, "accuracy_std_dev": 0.016330067701601517, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03858049, "avg_duration_per_task_ms": 137271.52, "avg_output_tokens_per_task": 3487.26, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.4328860536912752, "accuracy_std_dev": 0.30245117330901533, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.006059127516778524, "avg_duration_per_task_ms": 401903.8154362416, "avg_output_tokens_per_task": 13127.23154362416, "last_run_timestamp": "2026-10-24T19:32:27.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.4327958548387097, "accuracy_std_dev": 0.02500116609760433, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.016440024193548387, "avg_duration_per_task_ms": 245781.88709677418, "avg_output_tokens_per_task": 2789.4475806451615, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4321305463917526, "accuracy_std_dev": 0.024017877565471933, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.03244629381443299, "avg_duration_per_task_ms": 504849.24742268043, "avg_output_tokens_per_task": 4749.7525773195875, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.43111116666666666, "accuracy_std_dev": 0.04833021190524159, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4555577033333333, "avg_duration_per_task_ms": 109357.17666666667, "avg_output_tokens_per_task": 3283.1, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.431111, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029769013333333334, "avg_duration_per_task_ms": 630152.1266666667, "avg_output_tokens_per_task": 18760.18666666667, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.43089426016260163, "accuracy_std_dev": 0.03483555910043644, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.04202976422764227, "avg_duration_per_task_ms": 58240.52032520325, "avg_output_tokens_per_task": 2134.682926829268, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.43083325, "accuracy_std_dev": 0.031213896924086562, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029063365, "avg_duration_per_task_ms": 761962.965, "avg_output_tokens_per_task": 19001.86, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.24811575000000002, "avg_duration_per_task_ms": 46287.18, "avg_output_tokens_per_task": 2565.8, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.4293334, "accuracy_std_dev": 0.013727396775791108, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046574660000000004, "avg_duration_per_task_ms": 138519.112, "avg_output_tokens_per_task": 2139.824, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.42916675, "accuracy_std_dev": 0.025967332577442378, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027418319999999996, "avg_duration_per_task_ms": 84447.825, "avg_output_tokens_per_task": 2784.11, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.42795377521613837, "accuracy_std_dev": 0.029431010421121517, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.05299580115273775, "avg_duration_per_task_ms": 466627.5331412104, "avg_output_tokens_per_task": 21231.06916426513, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.42555566666666667, "accuracy_std_dev": 0.013698558095248165, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02489071, "avg_duration_per_task_ms": 220083.76333333334, "avg_output_tokens_per_task": 3565.5666666666666, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.4226667, "accuracy_std_dev": 0.03237967195031475, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.024325878, "avg_duration_per_task_ms": 394315.086, "avg_output_tokens_per_task": 3598.632, "last_run_timestamp": "2026-10-24T19:25:31.387Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.42006044796380093, "accuracy_std_dev": 0.026085942399861443, "run_count": 9, "total_questions": 442, "avg_cost_per_task_usd": 0.05284868552036199, "avg_duration_per_task_ms": 192763.79411764705, "avg_output_tokens_per_task": 4184.2692307692305, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.41733359999999997, "accuracy_std_dev": 0.04591776804506073, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.054456984, "avg_duration_per_task_ms": 140659.208, "avg_output_tokens_per_task": 3410.888, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.41666675909090906, "accuracy_std_dev": 0.02610295346611798, "run_count": 9, "total_questions": 440, "avg_cost_per_task_usd": 0.03160882727272728, "avg_duration_per_task_ms": 641960.8022727272, "avg_output_tokens_per_task": 6320.127272727273, "last_run_timestamp": "2026-10-24T19:18:23.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03394864, "avg_duration_per_task_ms": 276713.14, "avg_output_tokens_per_task": 20490.4, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.410000125, "accuracy_std_dev": 0.05535228340239791, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.007014935, "avg_duration_per_task_ms": 163231.8775, "avg_output_tokens_per_task": 4221.6925, "last_run_timestamp": "2026-10-24T19:16:50.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.4099998333333334, "accuracy_std_dev": 0.01953144663029912, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06387897999999999, "avg_duration_per_task_ms": 359150.7866666667, "avg_output_tokens_per_task": 9115.776666666667, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4088235294117647, "accuracy_std_dev": 0.023166950988907642, "run_count": 7, "total_questions": 340, "avg_cost_per_task_usd": 0.011171617647058823, "avg_duration_per_task_ms": 111563.3294117647, "avg_output_tokens_per_task": 3213.9382352941175, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40777783333333334, "accuracy_std_dev": 0.01781244190087991, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025491406666666664, "avg_duration_per_task_ms": 73232.91333333333, "avg_output_tokens_per_task": 2860.7966666666666, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.4053599698492462, "accuracy_std_dev": 0.008101744842933528, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.06672347236180905, "avg_duration_per_task_ms": 212067.50753768845, "avg_output_tokens_per_task": 8080.718592964824, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.40389273722627733, "accuracy_std_dev": 0.01580206672840261, "run_count": 3, "total_questions": 137, "avg_cost_per_task_usd": 0.008183423357664232, "avg_duration_per_task_ms": 89294.24087591241, "avg_output_tokens_per_task": 1500.5474452554745, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.4033333333333333, "accuracy_std_dev": 0.032603090061458215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07132451, "avg_duration_per_task_ms": 337606.7366666667, "avg_output_tokens_per_task": 13788.606666666667, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.4000001666666666, "accuracy_std_dev": 0.025819845942069878, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.027428286666666666, "avg_duration_per_task_ms": 327766.04, "avg_output_tokens_per_task": 4214.8533333333335, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.3999998333333334, "accuracy_std_dev": 0.027755373188007314, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07644987333333333, "avg_duration_per_task_ms": 118340.34, "avg_output_tokens_per_task": 3090.1866666666665, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.3990239504504504, "accuracy_std_dev": 0.05792485239888057, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.004533889639639639, "avg_duration_per_task_ms": 307259.5382882883, "avg_output_tokens_per_task": 4442.5472972972975, "last_run_timestamp": "2026-10-24T19:54:38.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.3983335, "accuracy_std_dev": 0.01960272562468461, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.062113629999999996, "avg_duration_per_task_ms": 98902.40333333334, "avg_output_tokens_per_task": 2914.5433333333335, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.39555566666666664, "accuracy_std_dev": 0.058203726918662366, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05201909999999999, "avg_duration_per_task_ms": 91142.84333333334, "avg_output_tokens_per_task": 3810.85, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.39555533333333337, "accuracy_std_dev": 0.012570885432440924, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03699347666666667, "avg_duration_per_task_ms": 190794.29666666666, "avg_output_tokens_per_task": 3701.5133333333333, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3948148888888889, "accuracy_std_dev": 0.1204017290208487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.023600613333333336, "avg_duration_per_task_ms": 193026.64, "avg_output_tokens_per_task": 3368.5844444444447, "last_run_timestamp": "2026-10-24T19:36:27.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.3944446666666666, "accuracy_std_dev": 0.015947545189011245, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07351260000000001, "avg_duration_per_task_ms": 297595.3333333333, "avg_output_tokens_per_task": 7950.953333333333, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.39210027167630057, "accuracy_std_dev": 0.08499299536756556, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.3060756098265896, "avg_duration_per_task_ms": 324526.6416184971, "avg_output_tokens_per_task": 12470.086705202311, "last_run_timestamp": "2026-10-10T18:31:45.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.39, "accuracy_std_dev": 0.019148600183825446, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.025881283333333335, "avg_duration_per_task_ms": 240617.67666666667, "avg_output_tokens_per_task": 4326.496666666667, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.3893334, "accuracy_std_dev": 0.009043086986201126, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014810912, "avg_duration_per_task_ms": 96843.704, "avg_output_tokens_per_task": 601.908, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3868015260115607, "accuracy_std_dev": 0.026163281770060737, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.05894507803468208, "avg_duration_per_task_ms": 398637.9768786127, "avg_output_tokens_per_task": 3843.208092485549, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.384, "accuracy_std_dev": 0.006798758063058267, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.028780152000000003, "avg_duration_per_task_ms": 81353.024, "avg_output_tokens_per_task": 4284.144, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.3759873436018957, "accuracy_std_dev": 0.1400803380925799, "run_count": 17, "total_questions": 844, "avg_cost_per_task_usd": 0.03816363151658768, "avg_duration_per_task_ms": 580869.5746445497, "avg_output_tokens_per_task": 145902.23341232227, "last_run_timestamp": "2026-10-24T19:41:04.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.3737958670520231, "accuracy_std_dev": 0.32322783310601233, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.012894294797687859, "avg_duration_per_task_ms": 78253.61560693642, "avg_output_tokens_per_task": 13434.153179190751, "last_run_timestamp": "2026-10-24T19:09:36.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.37205161949265686, "accuracy_std_dev": 0.021985652621046604, "run_count": 15, "total_questions": 749, "avg_cost_per_task_usd": 0.013301118825100131, "avg_duration_per_task_ms": 38508.31642189586, "avg_output_tokens_per_task": 742.2656875834446, "last_run_timestamp": "2026-10-24T18:51:12.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.36999983333333336, "accuracy_std_dev": 0.032375053268510444, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05485019333333334, "avg_duration_per_task_ms": 70677.64666666667, "avg_output_tokens_per_task": 4390.706666666667, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.3688886666666667, "accuracy_std_dev": 0.01791611454963257, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03368235333333333, "avg_duration_per_task_ms": 150468.61333333334, "avg_output_tokens_per_task": 4460.053333333333, "last_run_timestamp": "2026-10-23T08:14:36.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.363333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03867836, "avg_duration_per_task_ms": 117421.08, "avg_output_tokens_per_task": 4043.36, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35752687096774194, "accuracy_std_dev": 0.02846485415894485, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.07167372177419354, "avg_duration_per_task_ms": 382176.28830645164, "avg_output_tokens_per_task": 7777.157258064516, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.3555554444444445, "accuracy_std_dev": 0.0331101789589147, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08666491777777777, "avg_duration_per_task_ms": 26232.928888888888, "avg_output_tokens_per_task": 1004.2177777777778, "last_run_timestamp": "2026-10-24T18:19:37.955Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.3519345982142857, "accuracy_std_dev": 0.02099059885040837, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07758646428571427, "avg_duration_per_task_ms": 205456.72098214287, "avg_output_tokens_per_task": 7279.627232142857, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03045268, "avg_duration_per_task_ms": 77570.34, "avg_output_tokens_per_task": 3577.76, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.32660695515695065, "accuracy_std_dev": 0.04329336143048513, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.04019819282511211, "avg_duration_per_task_ms": 95760.30493273542, "avg_output_tokens_per_task": 1304.5695067264573, "last_run_timestamp": "2026-10-24T18:56:17.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.325926, "accuracy_std_dev": 0.02483419696484847, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07731095111111112, "avg_duration_per_task_ms": 138830.34, "avg_output_tokens_per_task": 7809.953333333333, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.30874319672131145, "accuracy_std_dev": 0.03698276832472119, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.01523452049180328, "avg_duration_per_task_ms": 155396.16393442624, "avg_output_tokens_per_task": 1405.610655737705, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.29333325000000005, "accuracy_std_dev": 0.03543382721619978, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1708829475, "avg_duration_per_task_ms": 140129.865, "avg_output_tokens_per_task": 1891.19, "last_run_timestamp": "2026-10-17T19:19:08.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.2844443333333333, "accuracy_std_dev": 0.031894788459698065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02081186, "avg_duration_per_task_ms": 62184.12666666666, "avg_output_tokens_per_task": 3804.7533333333336, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.28000025, "accuracy_std_dev": 0.027080189561513406, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02535222, "avg_duration_per_task_ms": 67904.845, "avg_output_tokens_per_task": 4023.05, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2622223333333333, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08337879999999999, "avg_duration_per_task_ms": 576772.3666666667, "avg_output_tokens_per_task": 5220.566666666667, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.25879402512562816, "accuracy_std_dev": 0.19158936601948973, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.1755232135678392, "avg_duration_per_task_ms": 1051691.3015075377, "avg_output_tokens_per_task": 6720.771356783919, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.2550507121212121, "accuracy_std_dev": 0.24463597214378424, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.010489681818181816, "avg_duration_per_task_ms": 1138281.7373737374, "avg_output_tokens_per_task": 70411.75252525252, "last_run_timestamp": "2026-10-24T19:33:50.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.2513333, "accuracy_std_dev": 0.2375252245788013, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.079624326, "avg_duration_per_task_ms": 183949.264, "avg_output_tokens_per_task": 23168.86, "last_run_timestamp": "2026-10-24T19:07:57.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24200919178082192, "accuracy_std_dev": 0.027381797668443095, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.07592458219178082, "avg_duration_per_task_ms": 576526.0342465753, "avg_output_tokens_per_task": 5262.068493150685, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.22555566666666663, "accuracy_std_dev": 0.06975256621722879, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01210011333333333, "avg_duration_per_task_ms": 20112.93, "avg_output_tokens_per_task": 1002.5833333333334, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.22244883673469384, "accuracy_std_dev": 0.060049660254159636, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.004490742857142857, "avg_duration_per_task_ms": 16706.061224489797, "avg_output_tokens_per_task": 1551.4979591836734, "last_run_timestamp": "2026-10-24T19:30:38.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.22242427272727275, "accuracy_std_dev": 0.29734520032174755, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.010851310909090908, "avg_duration_per_task_ms": 29224.176363636365, "avg_output_tokens_per_task": 5625.745454545455, "last_run_timestamp": "2026-10-17T19:48:24.252Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.2213332, "accuracy_std_dev": 0.11472869795025131, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.017538988, "avg_duration_per_task_ms": 77819.968, "avg_output_tokens_per_task": 1398.036, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.2066666, "accuracy_std_dev": 0.10724846748480837, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.19412482799999997, "avg_duration_per_task_ms": 581641.784, "avg_output_tokens_per_task": 11075.26, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.18766967073170732, "accuracy_std_dev": 0.018396157170452745, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.011055426829268291, "avg_duration_per_task_ms": 138140.34552845528, "avg_output_tokens_per_task": 5390.59756097561, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.13333333333333333, "accuracy_std_dev": 0.021430473285072877, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007961373333333334, "avg_duration_per_task_ms": 80728.30333333333, "avg_output_tokens_per_task": 2469.17, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.11833325, "accuracy_std_dev": 0.018484159885899604, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007158584999999999, "avg_duration_per_task_ms": 55526.67, "avg_output_tokens_per_task": 2353.445, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.108889, "accuracy_std_dev": 0.008314764498569193, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006775626666666668, "avg_duration_per_task_ms": 60620.74, "avg_output_tokens_per_task": 2327.0066666666667, "last_run_timestamp": "2026-10-01T19:04:36.194Z" } ] }