{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-11-07", "retrieved_at": "2026-11-07T01:45:34.869Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2263, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.964646, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09468125757575757, "avg_duration_per_task_ms": 101980.28282828283, "avg_output_tokens_per_task": 8439.414141414141, "last_run_timestamp": "2026-10-14T15:00:03.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9511779999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20247136363636364, "avg_duration_per_task_ms": 137578.30555555556, "avg_output_tokens_per_task": 16748.56818181818, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.9482614885496183, "accuracy_std_dev": 0.0012429999999999941, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07536056743002545, "avg_duration_per_task_ms": 132814.94147582698, "avg_output_tokens_per_task": 18975.51145038168, "last_run_timestamp": "2026-10-15T21:00:04.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.947917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 96, "avg_cost_per_task_usd": 0.009331666666666667, "avg_duration_per_task_ms": 232878.71875, "avg_output_tokens_per_task": 10278.75, "last_run_timestamp": "2026-10-14T18:00:04.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021484267676767678, "avg_duration_per_task_ms": 55623.856060606064, "avg_output_tokens_per_task": 1890.328282828283, "last_run_timestamp": "2026-10-30T00:03:33.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.9461276666666666, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1181814292929293, "avg_duration_per_task_ms": 36670.56565656565, "avg_output_tokens_per_task": 2059.811447811448, "last_run_timestamp": "2026-10-30T00:02:55.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.941077, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.15125092424242426, "avg_duration_per_task_ms": 97603.77272727272, "avg_output_tokens_per_task": 11559.585858585859, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Nebius", "accuracy": 0.939394, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06875348484848484, "avg_duration_per_task_ms": 279532.6212121212, "avg_output_tokens_per_task": 6669.813131313132, "last_run_timestamp": "2026-11-04T21:00:08.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.31252995454545457, "avg_duration_per_task_ms": 150832.55050505052, "avg_output_tokens_per_task": 10288.712121212122, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.32760574494949496, "avg_duration_per_task_ms": 78455.45202020202, "avg_output_tokens_per_task": 11977.780303030304, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9393935, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15067959090909092, "avg_duration_per_task_ms": 95544.60353535354, "avg_output_tokens_per_task": 11515.699494949495, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9391951653543308, "accuracy_std_dev": 0.009884880879403648, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.026313035433070862, "avg_duration_per_task_ms": 56131.107611548556, "avg_output_tokens_per_task": 7729.320209973754, "last_run_timestamp": "2026-10-15T21:01:11.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Sail Research", "accuracy": 0.9387350223978919, "accuracy_std_dev": 0.0014398627538762172, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.14429597496706192, "avg_duration_per_task_ms": 118270.40579710146, "avg_output_tokens_per_task": 6705.238471673254, "last_run_timestamp": "2026-11-06T22:04:38.207Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9336215714285715, "accuracy_std_dev": 0.008810277924222438, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.3281882835497835, "avg_duration_per_task_ms": 164668.6948051948, "avg_output_tokens_per_task": 10091.254689754689, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Io Net", "accuracy": 0.933333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.10421853333333334, "avg_duration_per_task_ms": 136414.0358974359, "avg_output_tokens_per_task": 6800.2, "last_run_timestamp": "2026-11-04T16:38:30.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Modal", "accuracy": 0.931818, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07295092424242425, "avg_duration_per_task_ms": 39086.47474747475, "avg_output_tokens_per_task": 7118.949494949495, "last_run_timestamp": "2026-11-04T21:00:08.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.9309762, "accuracy_std_dev": 0.005106374267520954, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.057293524242424246, "avg_duration_per_task_ms": 66775.28888888888, "avg_output_tokens_per_task": 11798.8, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Parasail", "accuracy": 0.930976, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07401931818181819, "avg_duration_per_task_ms": 70607.26262626263, "avg_output_tokens_per_task": 7194.131313131313, "last_run_timestamp": "2026-11-04T21:00:08.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "InferenceNet", "accuracy": 0.9306844829721364, "accuracy_std_dev": 0.007658821616932981, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.0836984004127967, "avg_duration_per_task_ms": 71158.75335397317, "avg_output_tokens_per_task": 7439.702786377709, "last_run_timestamp": "2026-11-05T00:00:09.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9301345000000001, "accuracy_std_dev": 0.006832688313053163, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.026123152076318747, "avg_duration_per_task_ms": 18695.293490460157, "avg_output_tokens_per_task": 2221.0524691358023, "last_run_timestamp": "2026-10-31T08:07:18.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9301345, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044492378787878796, "avg_duration_per_task_ms": 69170.04040404041, "avg_output_tokens_per_task": 11715.780303030304, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.929854, "accuracy_std_dev": 0.006348890821762996, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.025367877946127944, "avg_duration_per_task_ms": 20154.14898989899, "avg_output_tokens_per_task": 2304.1734006734005, "last_run_timestamp": "2026-10-31T08:33:04.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Moonshot AI", "accuracy": 0.9292930000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10512464646464648, "avg_duration_per_task_ms": 151923.42676767678, "avg_output_tokens_per_task": 6756.267676767677, "last_run_timestamp": "2026-11-04T18:03:37.503Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "AkashML", "accuracy": 0.9292929999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08810357575757576, "avg_duration_per_task_ms": 85562.65151515152, "avg_output_tokens_per_task": 8602.494949494949, "last_run_timestamp": "2026-11-05T06:00:43.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Morph", "accuracy": 0.9292929999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11162276262626263, "avg_duration_per_task_ms": 129833.19191919192, "avg_output_tokens_per_task": 9183.469696969696, "last_run_timestamp": "2026-11-05T06:00:06.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.9290121666666666, "accuracy_std_dev": 0.0044978590987515684, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024844954545454545, "avg_duration_per_task_ms": 19502.118686868685, "avg_output_tokens_per_task": 2244.3215488215487, "last_run_timestamp": "2026-10-31T09:02:13.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.928451, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.3476244823232323, "avg_duration_per_task_ms": 112460.52777777778, "avg_output_tokens_per_task": 10405.60101010101, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.9282826, "accuracy_std_dev": 0.00626296053316642, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025195250505050507, "avg_duration_per_task_ms": 19633.72121212121, "avg_output_tokens_per_task": 2281.29898989899, "last_run_timestamp": "2026-10-31T08:09:30.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Tenstorrent", "accuracy": 0.9281247875, "accuracy_std_dev": 0.003304499999999988, "run_count": 2, "total_questions": 160, "avg_cost_per_task_usd": 0.01232809375, "avg_duration_per_task_ms": 251386.81875, "avg_output_tokens_per_task": 10178.75625, "last_run_timestamp": "2026-11-04T18:00:19.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9276095000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04436604545454545, "avg_duration_per_task_ms": 61670.28282828283, "avg_output_tokens_per_task": 11683.290404040405, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9276091428571428, "accuracy_std_dev": 0.005010400338398178, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10054259307359308, "avg_duration_per_task_ms": 55310.95093795094, "avg_output_tokens_per_task": 3496.1031746031745, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Fireworks", "accuracy": 0.927609, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07610462626262626, "avg_duration_per_task_ms": 98167.92929292929, "avg_output_tokens_per_task": 7400.540404040404, "last_run_timestamp": "2026-11-05T15:00:03.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9275846571787361, "accuracy_std_dev": 0.010407425048021134, "run_count": 13, "total_questions": 2389, "avg_cost_per_task_usd": 0.13203736626203433, "avg_duration_per_task_ms": 128986.54792800335, "avg_output_tokens_per_task": 8711.713269150272, "last_run_timestamp": "2026-10-31T08:39:06.851Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "PrimaLabs", "accuracy": 0.927326, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.04753577906976744, "avg_duration_per_task_ms": 65225.773255813954, "avg_output_tokens_per_task": 5210.343023255814, "last_run_timestamp": "2026-11-06T15:43:35.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Phala", "accuracy": 0.9267675, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07679769191919193, "avg_duration_per_task_ms": 175516.4292929293, "avg_output_tokens_per_task": 7530.295454545455, "last_run_timestamp": "2026-11-05T18:00:48.183Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.007815979699734826, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03737200673400674, "avg_duration_per_task_ms": 60770.02693602694, "avg_output_tokens_per_task": 3580.875420875421, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9264542826738137, "accuracy_std_dev": 0.007871563573392863, "run_count": 34, "total_questions": 6343, "avg_cost_per_task_usd": 0.1726583693835724, "avg_duration_per_task_ms": 167044.34431656945, "avg_output_tokens_per_task": 9360.229386725525, "last_run_timestamp": "2026-10-31T08:52:06.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9260940999999999, "accuracy_std_dev": 0.00286181765491794, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.026339350505050502, "avg_duration_per_task_ms": 19433.036868686868, "avg_output_tokens_per_task": 2294.320202020202, "last_run_timestamp": "2026-10-31T08:17:29.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9259260000000001, "accuracy_std_dev": 0.004610436378478734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10788488484848485, "avg_duration_per_task_ms": 187765.80707070706, "avg_output_tokens_per_task": 9499.632323232323, "last_run_timestamp": "2026-10-24T08:37:09.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9255047500000002, "accuracy_std_dev": 0.0055038172832589756, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.14877371590909091, "avg_duration_per_task_ms": 85367.24242424243, "avg_output_tokens_per_task": 15632.513888888889, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.9252448891402715, "accuracy_std_dev": 0.0069132275216017836, "run_count": 9, "total_questions": 1768, "avg_cost_per_task_usd": 0.10858752149321266, "avg_duration_per_task_ms": 188184.25339366516, "avg_output_tokens_per_task": 9458.231334841628, "last_run_timestamp": "2026-10-31T09:20:20.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Darkbloom", "accuracy": 0.925214, "accuracy_std_dev": null, "run_count": 1, "total_questions": 78, "avg_cost_per_task_usd": 0.0012695128205128204, "avg_duration_per_task_ms": 237789.78205128206, "avg_output_tokens_per_task": 6844.423076923077, "last_run_timestamp": "2026-10-30T01:38:27.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9244886751858735, "accuracy_std_dev": 0.010749601793658319, "run_count": 12, "total_questions": 2152, "avg_cost_per_task_usd": 0.14484657992565056, "avg_duration_per_task_ms": 111589.46514869889, "avg_output_tokens_per_task": 9509.516728624536, "last_run_timestamp": "2026-10-31T08:14:43.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9243079168138753, "accuracy_std_dev": 0.02419971284017385, "run_count": 32, "total_questions": 6227, "avg_cost_per_task_usd": 0.1328945098763449, "avg_duration_per_task_ms": 158997.73036775333, "avg_output_tokens_per_task": 10041.62630480167, "last_run_timestamp": "2026-11-04T21:14:13.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Together", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09054113636363637, "avg_duration_per_task_ms": 151399.2323232323, "avg_output_tokens_per_task": 8895.065656565657, "last_run_timestamp": "2026-11-04T18:00:08.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03863523484848485, "avg_duration_per_task_ms": 63161.229797979795, "avg_output_tokens_per_task": 3708.2954545454545, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9240912983587339, "accuracy_std_dev": 0.015687164131543977, "run_count": 10, "total_questions": 1706, "avg_cost_per_task_usd": 0.1119299531066823, "avg_duration_per_task_ms": 198230.14243845252, "avg_output_tokens_per_task": 8354.249120750294, "last_run_timestamp": "2026-10-31T08:14:43.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Near AI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02620280303030303, "avg_duration_per_task_ms": 592141.2247474748, "avg_output_tokens_per_task": 21737.315656565657, "last_run_timestamp": "2026-10-31T10:06:11.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1436656515151515, "avg_duration_per_task_ms": 89621.648989899, "avg_output_tokens_per_task": 15839.800505050505, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.9228069789473685, "accuracy_std_dev": 0.011126928806578514, "run_count": 3, "total_questions": 570, "avg_cost_per_task_usd": 0.00580749649122807, "avg_duration_per_task_ms": 218614.69122807018, "avg_output_tokens_per_task": 19521.97543859649, "last_run_timestamp": "2026-10-31T08:40:34.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.9226093316749586, "accuracy_std_dev": 0.014318661477247108, "run_count": 4, "total_questions": 603, "avg_cost_per_task_usd": 0.01542157711442786, "avg_duration_per_task_ms": 482101.75787728024, "avg_output_tokens_per_task": 30680.03648424544, "last_run_timestamp": "2026-11-04T18:05:51.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.92256925, "accuracy_std_dev": 0.005618592528382903, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.148641165625, "avg_duration_per_task_ms": 385822.7, "avg_output_tokens_per_task": 9697.144791666668, "last_run_timestamp": "2026-10-31T08:38:36.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9225589999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14588472727272728, "avg_duration_per_task_ms": 63933.833333333336, "avg_output_tokens_per_task": 16085.368686868687, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9224701578947367, "accuracy_std_dev": 0.007214706969744515, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.13204979505582137, "avg_duration_per_task_ms": 175028.0927698033, "avg_output_tokens_per_task": 11401.640882509304, "last_run_timestamp": "2026-11-06T22:04:48.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9219976666666666, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.08309612457912459, "avg_duration_per_task_ms": 49364.952861952865, "avg_output_tokens_per_task": 3748.658249158249, "last_run_timestamp": "2026-10-30T00:00:06.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.9208753333333334, "accuracy_std_dev": 0.003636621356271357, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019656516835016836, "avg_duration_per_task_ms": 90437.9696969697, "avg_output_tokens_per_task": 20830.57239057239, "last_run_timestamp": "2026-10-31T08:09:30.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026268675084175085, "avg_duration_per_task_ms": 25466.75420875421, "avg_output_tokens_per_task": 2308.39898989899, "last_run_timestamp": "2026-10-30T00:00:05.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.9205386, "accuracy_std_dev": 0.004178401589124734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.14947429494949496, "avg_duration_per_task_ms": 120588.33434343434, "avg_output_tokens_per_task": 9732.809090909092, "last_run_timestamp": "2026-10-31T08:09:30.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9200085693112466, "accuracy_std_dev": 0.012700463146629303, "run_count": 12, "total_questions": 2294, "avg_cost_per_task_usd": 0.13002592894507412, "avg_duration_per_task_ms": 114105.1320836966, "avg_output_tokens_per_task": 8645, "last_run_timestamp": "2026-10-31T08:14:19.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.9199569999999999, "accuracy_std_dev": 0.006554291127331654, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.17177113406795225, "avg_duration_per_task_ms": 299167.57943067036, "avg_output_tokens_per_task": 9764.863636363636, "last_run_timestamp": "2026-10-31T08:33:15.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Wafer", "accuracy": 0.9191920000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07685085101010102, "avg_duration_per_task_ms": 125614, "avg_output_tokens_per_task": 7560.146464646465, "last_run_timestamp": "2026-11-05T21:03:19.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Relace", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.062148696969696965, "avg_duration_per_task_ms": 92710.41414141415, "avg_output_tokens_per_task": 7696.424242424242, "last_run_timestamp": "2026-11-05T00:00:09.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.7-20260916", "model_slug": "x-ai/grok-4.7", "model_name": "SpaceXAI: Grok 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.21825907575757575, "avg_duration_per_task_ms": 803866.0202020202, "avg_output_tokens_per_task": 36106.61616161616, "last_run_timestamp": "2026-11-03T03:00:04.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9187325454545454, "accuracy_std_dev": 0.01067374820732572, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12858450137741045, "avg_duration_per_task_ms": 254711.14095500458, "avg_output_tokens_per_task": 10409.111570247935, "last_run_timestamp": "2026-10-31T08:21:51.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.9186305, "accuracy_std_dev": 0.0030220059425267055, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.012602135521885521, "avg_duration_per_task_ms": 20976.548821548822, "avg_output_tokens_per_task": 2132.777777777778, "last_run_timestamp": "2026-11-02T17:19:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.9186303333333333, "accuracy_std_dev": 0.010408091158111372, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025292042087542085, "avg_duration_per_task_ms": 160402.3451178451, "avg_output_tokens_per_task": 20957.713804713803, "last_run_timestamp": "2026-10-31T08:50:50.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9177144177922683, "accuracy_std_dev": 0.013751154902061724, "run_count": 11, "total_questions": 2147, "avg_cost_per_task_usd": 0.2460820894271076, "avg_duration_per_task_ms": 409070.95482068, "avg_output_tokens_per_task": 37154.30647414998, "last_run_timestamp": "2026-10-31T08:34:22.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimaLabs", "accuracy": 0.9175825384615385, "accuracy_std_dev": 0.009135000000000004, "run_count": 2, "total_questions": 182, "avg_cost_per_task_usd": 0.03656871978021978, "avg_duration_per_task_ms": 44231.42857142857, "avg_output_tokens_per_task": 10074.60989010989, "last_run_timestamp": "2026-11-06T15:33:21.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9175082499999999, "accuracy_std_dev": 0.003571399938516188, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.025937152356902356, "avg_duration_per_task_ms": 19174.19191919192, "avg_output_tokens_per_task": 2235.470117845118, "last_run_timestamp": "2026-10-31T08:20:35.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.916877387341772, "accuracy_std_dev": 0.005308941032719149, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.10198255443037975, "avg_duration_per_task_ms": 806625.7974683545, "avg_output_tokens_per_task": 39607.979746835445, "last_run_timestamp": "2026-10-31T16:24:40.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.916859788863109, "accuracy_std_dev": 0.00696582562310962, "run_count": 11, "total_questions": 2155, "avg_cost_per_task_usd": 0.1413055136890951, "avg_duration_per_task_ms": 281514.6403712297, "avg_output_tokens_per_task": 9248.70162412993, "last_run_timestamp": "2026-10-24T08:44:42.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9162757342365815, "accuracy_std_dev": 0.00785699694603481, "run_count": 10, "total_questions": 1919, "avg_cost_per_task_usd": 0.09128974934861907, "avg_duration_per_task_ms": 645077.3731109953, "avg_output_tokens_per_task": 39631.607087024495, "last_run_timestamp": "2026-10-31T08:24:03.784Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9158379972375691, "accuracy_std_dev": 0.012441503983078046, "run_count": 13, "total_questions": 1810, "avg_cost_per_task_usd": 0.11735693314917128, "avg_duration_per_task_ms": 179712.15801104973, "avg_output_tokens_per_task": 8118.493370165746, "last_run_timestamp": "2026-10-31T08:07:17.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DekaLLM", "accuracy": 0.915825, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014663378787878788, "avg_duration_per_task_ms": 96926.35858585859, "avg_output_tokens_per_task": 21419.76767676768, "last_run_timestamp": "2026-11-04T09:00:09.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "DeepInfra", "accuracy": 0.915825, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06562552525252525, "avg_duration_per_task_ms": 514778.64646464644, "avg_output_tokens_per_task": 6703.651515151515, "last_run_timestamp": "2026-11-05T06:03:33.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9154666900732507, "accuracy_std_dev": 0.026006001110512943, "run_count": 20, "total_questions": 3959, "avg_cost_per_task_usd": 0.14730550719878757, "avg_duration_per_task_ms": 390507.92093963124, "avg_output_tokens_per_task": 9397.29047739328, "last_run_timestamp": "2026-10-31T09:12:10.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9149830000000001, "accuracy_std_dev": 0.009335323856192691, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15949898674242427, "avg_duration_per_task_ms": 579778.1357323233, "avg_output_tokens_per_task": 39960.98863636364, "last_run_timestamp": "2026-10-31T08:49:58.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "CoreWeave", "accuracy": 0.9149660000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.022535153061224493, "avg_duration_per_task_ms": 106164.53571428571, "avg_output_tokens_per_task": 18639.61224489796, "last_run_timestamp": "2026-11-04T18:00:08.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9148920332741792, "accuracy_std_dev": 0.014548429093080425, "run_count": 12, "total_questions": 2254, "avg_cost_per_task_usd": 0.16241754968944103, "avg_duration_per_task_ms": 496269.2985803017, "avg_output_tokens_per_task": 11219.274622892635, "last_run_timestamp": "2026-10-31T08:37:30.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.914552902173913, "accuracy_std_dev": 0.010001261731512806, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.01860343115942029, "avg_duration_per_task_ms": 136525.615942029, "avg_output_tokens_per_task": 19948.84963768116, "last_run_timestamp": "2026-10-31T08:12:46.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9134399166666667, "accuracy_std_dev": 0.013804946332494593, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.033723330808080806, "avg_duration_per_task_ms": 222484.18686868687, "avg_output_tokens_per_task": 27941.286616161615, "last_run_timestamp": "2026-10-31T16:05:09.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9129648851570964, "accuracy_std_dev": 0.008195106272424223, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.0985225175153485, "avg_duration_per_task_ms": 283575.7085590466, "avg_output_tokens_per_task": 25018.89021307331, "last_run_timestamp": "2026-10-31T08:40:32.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.91245775, "accuracy_std_dev": 0.005709247427419856, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.027895760101010105, "avg_duration_per_task_ms": 177535.13888888888, "avg_output_tokens_per_task": 21414.281565656565, "last_run_timestamp": "2026-10-31T08:15:24.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9119986363636363, "accuracy_std_dev": 0.011140273942059605, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03546824609733701, "avg_duration_per_task_ms": 297176.681359045, "avg_output_tokens_per_task": 29467.73370064279, "last_run_timestamp": "2026-10-31T16:17:15.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.911926105263158, "accuracy_std_dev": 0.013578898854258508, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.1411968139287613, "avg_duration_per_task_ms": 616719.149122807, "avg_output_tokens_per_task": 40116.86735778841, "last_run_timestamp": "2026-10-31T08:31:11.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.911616, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.024379722222222223, "avg_duration_per_task_ms": 102982.82575757576, "avg_output_tokens_per_task": 20129.815656565657, "last_run_timestamp": "2026-10-31T08:34:49.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.910774, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.011462444444444444, "avg_duration_per_task_ms": 27557.444444444445, "avg_output_tokens_per_task": 2133.969696969697, "last_run_timestamp": "2026-10-28T00:00:04.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.9106891772151898, "accuracy_std_dev": 0.008562817793226714, "run_count": 5, "total_questions": 948, "avg_cost_per_task_usd": 0.14278316033755276, "avg_duration_per_task_ms": 333972.91455696203, "avg_output_tokens_per_task": 37406.36181434599, "last_run_timestamp": "2026-10-31T08:31:11.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.9103534999999999, "accuracy_std_dev": 0.003644869990822727, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.019048060606060607, "avg_duration_per_task_ms": 135272.13636363635, "avg_output_tokens_per_task": 22610.929292929293, "last_run_timestamp": "2026-10-31T08:28:22.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.9096519999999999, "accuracy_std_dev": 0.0075707498968068145, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012606855218855219, "avg_duration_per_task_ms": 20423.260942760942, "avg_output_tokens_per_task": 2135.06734006734, "last_run_timestamp": "2026-11-03T15:00:04.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.909632, "accuracy_std_dev": 0.0055275000000000185, "run_count": 2, "total_questions": 308, "avg_cost_per_task_usd": 0.05270355519480519, "avg_duration_per_task_ms": 55285.37662337662, "avg_output_tokens_per_task": 12143.461038961039, "last_run_timestamp": "2026-10-31T08:15:03.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9095115, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0416229191919192, "avg_duration_per_task_ms": 138307.48484848486, "avg_output_tokens_per_task": 27969.503787878788, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.9090910000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018701613636363638, "avg_duration_per_task_ms": 137003.63383838383, "avg_output_tokens_per_task": 22059.484848484848, "last_run_timestamp": "2026-10-31T08:36:23.829Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9090908333333333, "accuracy_std_dev": 0.008189992133017543, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03826549410774411, "avg_duration_per_task_ms": 134585.28703703705, "avg_output_tokens_per_task": 28750.484848484848, "last_run_timestamp": "2026-10-27T00:36:05.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StepFun", "accuracy": 0.9090908333333333, "accuracy_std_dev": 0.007076201038614481, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01709530134680135, "avg_duration_per_task_ms": 84455.94696969698, "avg_output_tokens_per_task": 21725.516835016835, "last_run_timestamp": "2026-11-02T17:24:31.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9086667275553827, "accuracy_std_dev": 0.04751984064982348, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.14066514496696458, "avg_duration_per_task_ms": 326890.27283326857, "avg_output_tokens_per_task": 10307.279440342014, "last_run_timestamp": "2026-10-31T09:37:23.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.9086299041331802, "accuracy_std_dev": 0.031330168891428464, "run_count": 9, "total_questions": 1742, "avg_cost_per_task_usd": 0.046619725602755444, "avg_duration_per_task_ms": 419278.7634902411, "avg_output_tokens_per_task": 14257.997129735935, "last_run_timestamp": "2026-10-31T16:08:56.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9083779032258064, "accuracy_std_dev": 0.022205862232752862, "run_count": 12, "total_questions": 2232, "avg_cost_per_task_usd": 0.1474674775985663, "avg_duration_per_task_ms": 555743.3785842294, "avg_output_tokens_per_task": 37926.89829749104, "last_run_timestamp": "2026-10-31T08:44:11.309Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.908249, "accuracy_std_dev": 0.006574437713599543, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.009681809974747475, "avg_duration_per_task_ms": 167110.30050505052, "avg_output_tokens_per_task": 18361.854166666668, "last_run_timestamp": "2026-11-04T21:05:58.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9081554444444445, "accuracy_std_dev": 0.008876832231665518, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.033613874298540965, "avg_duration_per_task_ms": 210737.89562289562, "avg_output_tokens_per_task": 27019.509539842875, "last_run_timestamp": "2026-10-31T08:40:16.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.9078013574468085, "accuracy_std_dev": 0.012928926018815361, "run_count": 3, "total_questions": 470, "avg_cost_per_task_usd": 0.01807536170212766, "avg_duration_per_task_ms": 357786.7425531915, "avg_output_tokens_per_task": 17947.382978723403, "last_run_timestamp": "2026-10-31T09:05:48.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9076662307692308, "accuracy_std_dev": 0.02122131075178129, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.15455806177156176, "avg_duration_per_task_ms": 600133.6076146077, "avg_output_tokens_per_task": 38964.08663558664, "last_run_timestamp": "2026-10-31T08:28:10.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9076534680851064, "accuracy_std_dev": 0.005694785441280666, "run_count": 6, "total_questions": 1128, "avg_cost_per_task_usd": 0.025873300531914892, "avg_duration_per_task_ms": 176651.64184397162, "avg_output_tokens_per_task": 28451.522163120568, "last_run_timestamp": "2026-10-31T08:42:09.092Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9074649821150856, "accuracy_std_dev": 0.007911648334003893, "run_count": 13, "total_questions": 2572, "avg_cost_per_task_usd": 0.030564498833592534, "avg_duration_per_task_ms": 392326.3246500778, "avg_output_tokens_per_task": 31615.309875583203, "last_run_timestamp": "2026-10-31T16:32:36.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.9074074999999998, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014508287878787877, "avg_duration_per_task_ms": 90086.95707070707, "avg_output_tokens_per_task": 21790.61111111111, "last_run_timestamp": "2026-10-31T08:13:11.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.9074072499999999, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025605553030303027, "avg_duration_per_task_ms": 96285.60858585859, "avg_output_tokens_per_task": 21160.45580808081, "last_run_timestamp": "2026-10-31T09:42:32.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.9071887349785409, "accuracy_std_dev": 0.014345789366918764, "run_count": 5, "total_questions": 932, "avg_cost_per_task_usd": 0.12532361909871245, "avg_duration_per_task_ms": 1379721.1190987125, "avg_output_tokens_per_task": 49889.66094420601, "last_run_timestamp": "2026-10-31T08:27:43.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9070474558880683, "accuracy_std_dev": 0.028977319190080154, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.0985262642829382, "avg_duration_per_task_ms": 564503.555771473, "avg_output_tokens_per_task": 39384.385542168675, "last_run_timestamp": "2026-10-31T08:39:28.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.9070292653061225, "accuracy_std_dev": 0.005723067708842884, "run_count": 3, "total_questions": 588, "avg_cost_per_task_usd": 0.021270954081632654, "avg_duration_per_task_ms": 117754.6887755102, "avg_output_tokens_per_task": 21116.049319727892, "last_run_timestamp": "2026-10-31T08:51:31.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.9062632644787644, "accuracy_std_dev": 0.007463126815174401, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.024727387387387386, "avg_duration_per_task_ms": 89200.08815958817, "avg_output_tokens_per_task": 20548.897683397685, "last_run_timestamp": "2026-10-31T08:46:47.079Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9061883101265825, "accuracy_std_dev": 0.03992056619739797, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.1382923181434599, "avg_duration_per_task_ms": 209596.56919831224, "avg_output_tokens_per_task": 9286.707594936708, "last_run_timestamp": "2026-10-31T08:40:21.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.9061447499999998, "accuracy_std_dev": 0.005102889738912664, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007892248737373738, "avg_duration_per_task_ms": 123987.29040404041, "avg_output_tokens_per_task": 16306.166666666666, "last_run_timestamp": "2026-10-31T08:53:36.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9060041666666666, "accuracy_std_dev": 0.006179339377222198, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.032878036195286196, "avg_duration_per_task_ms": 809018.2592592592, "avg_output_tokens_per_task": 37448.21212121212, "last_run_timestamp": "2026-10-31T08:23:27.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.9059643571428573, "accuracy_std_dev": 0.02530737982319202, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.039628380591630596, "avg_duration_per_task_ms": 275120.5137085137, "avg_output_tokens_per_task": 30935.2702020202, "last_run_timestamp": "2026-10-31T16:23:41.132Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.9059342499999998, "accuracy_std_dev": 0.010910172663874526, "run_count": 3, "total_questions": 528, "avg_cost_per_task_usd": 0.008819143939393941, "avg_duration_per_task_ms": 163195.54924242425, "avg_output_tokens_per_task": 22179.03787878788, "last_run_timestamp": "2026-10-31T08:49:09.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9057238888888889, "accuracy_std_dev": 0.009356618102764875, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03432713860830528, "avg_duration_per_task_ms": 196757.96408529743, "avg_output_tokens_per_task": 28444.87317620651, "last_run_timestamp": "2026-10-31T08:29:56.453Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.9057234999999999, "accuracy_std_dev": 0.0029159075345422127, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013535670454545456, "avg_duration_per_task_ms": 88017.25757575757, "avg_output_tokens_per_task": 21462.573232323233, "last_run_timestamp": "2026-11-01T12:00:08.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.9054373465045593, "accuracy_std_dev": 0.0037554479359991244, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.026207787234042552, "avg_duration_per_task_ms": 127141.36068895644, "avg_output_tokens_per_task": 21741.29989868288, "last_run_timestamp": "2026-10-31T08:31:11.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9053496666666667, "accuracy_std_dev": 0.008612696635394351, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02068693490460157, "avg_duration_per_task_ms": 208793.50953984287, "avg_output_tokens_per_task": 28854.00224466891, "last_run_timestamp": "2026-10-31T09:01:06.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.9053028750000003, "accuracy_std_dev": 0.00682546092651441, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0031826717171717163, "avg_duration_per_task_ms": 242276.39330808082, "avg_output_tokens_per_task": 19735.85101010101, "last_run_timestamp": "2026-11-06T21:57:38.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9051541212505279, "accuracy_std_dev": 0.011635351751833228, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.0273217351077313, "avg_duration_per_task_ms": 321380.98436839884, "avg_output_tokens_per_task": 22616.132234896493, "last_run_timestamp": "2026-10-31T16:12:28.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.9049724513274336, "accuracy_std_dev": 0.008906991898924113, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.020146505689001264, "avg_duration_per_task_ms": 644404.02528445, "avg_output_tokens_per_task": 42703.32616940582, "last_run_timestamp": "2026-10-28T03:59:43.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9049466923076923, "accuracy_std_dev": 0.025579447099289023, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03460525097125097, "avg_duration_per_task_ms": 225805.94988344988, "avg_output_tokens_per_task": 28771.273892773894, "last_run_timestamp": "2026-10-31T16:04:48.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.9048818749999998, "accuracy_std_dev": 0.008666358814945001, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008252799242424242, "avg_duration_per_task_ms": 82042.59659090909, "avg_output_tokens_per_task": 16383.162878787878, "last_run_timestamp": "2026-10-31T08:56:01.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.9048634068157615, "accuracy_std_dev": 0.012266915351464668, "run_count": 5, "total_questions": 939, "avg_cost_per_task_usd": 0.06399653567625134, "avg_duration_per_task_ms": 75580.11395101172, "avg_output_tokens_per_task": 14917.831735889244, "last_run_timestamp": "2026-10-31T08:13:57.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9047416666666667, "accuracy_std_dev": 0.021088018854843207, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03692853914141414, "avg_duration_per_task_ms": 252506.9945286195, "avg_output_tokens_per_task": 30920.941498316497, "last_run_timestamp": "2026-10-31T16:09:19.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.904724560327997, "accuracy_std_dev": 0.021378149302998656, "run_count": 13, "total_questions": 2561, "avg_cost_per_task_usd": 0.12218845060523233, "avg_duration_per_task_ms": 327538.06130417803, "avg_output_tokens_per_task": 8367.566966028895, "last_run_timestamp": "2026-10-31T08:25:00.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.009152126540986068, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02685950505050505, "avg_duration_per_task_ms": 136981.2441077441, "avg_output_tokens_per_task": 22236.84175084175, "last_run_timestamp": "2026-10-31T08:31:11.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.003967811851498118, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.016546668350168353, "avg_duration_per_task_ms": 140240.16835016836, "avg_output_tokens_per_task": 21954.457912457914, "last_run_timestamp": "2026-10-31T08:27:53.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.008285549120138134, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01933321717171717, "avg_duration_per_task_ms": 139538.02861952863, "avg_output_tokens_per_task": 20900.63973063973, "last_run_timestamp": "2026-10-31T08:20:08.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9044004522003033, "accuracy_std_dev": 0.00856689234728672, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.014848743045017704, "avg_duration_per_task_ms": 370376.5558927668, "avg_output_tokens_per_task": 28855.582195245323, "last_run_timestamp": "2026-10-31T08:03:09.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9042316703786192, "accuracy_std_dev": 0.0031656583896560934, "run_count": 5, "total_questions": 898, "avg_cost_per_task_usd": 0.22350682405345215, "avg_duration_per_task_ms": 441723.66035634745, "avg_output_tokens_per_task": 42286.14922048998, "last_run_timestamp": "2026-10-31T08:44:55.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9041981924485772, "accuracy_std_dev": 0.01113877499359892, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.027435362073823615, "avg_duration_per_task_ms": 185052.6246830093, "avg_output_tokens_per_task": 25932.813750352212, "last_run_timestamp": "2026-10-31T08:28:13.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.9040405, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13174327272727274, "avg_duration_per_task_ms": 249918.9898989899, "avg_output_tokens_per_task": 43791.378787878784, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.9034793333333332, "accuracy_std_dev": 0.005722799567422317, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025755799663299665, "avg_duration_per_task_ms": 89661.28282828283, "avg_output_tokens_per_task": 21298.74074074074, "last_run_timestamp": "2026-10-31T09:51:08.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.9033668, "accuracy_std_dev": 0.005061737504059227, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.026716864646464646, "avg_duration_per_task_ms": 106368.8292929293, "avg_output_tokens_per_task": 22085.938383838384, "last_run_timestamp": "2026-10-31T08:13:57.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.9031985, "accuracy_std_dev": 0.0075759444460740165, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01504592297979798, "avg_duration_per_task_ms": 86737.82702020202, "avg_output_tokens_per_task": 17203.179292929293, "last_run_timestamp": "2026-11-03T01:49:06.293Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.9031985, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14226843434343436, "avg_duration_per_task_ms": 149341.0782828283, "avg_output_tokens_per_task": 9356.287878787878, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.9030760960809103, "accuracy_std_dev": 0.01701828861577222, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.10602527054361567, "avg_duration_per_task_ms": 544239.2713864307, "avg_output_tokens_per_task": 37272.33965444585, "last_run_timestamp": "2026-10-31T08:11:08.576Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.9029179999999999, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.023974107744107742, "avg_duration_per_task_ms": 50011.877104377105, "avg_output_tokens_per_task": 19809.430976430976, "last_run_timestamp": "2026-10-31T08:51:04.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9029179166666667, "accuracy_std_dev": 0.01824016717055307, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03467677272727272, "avg_duration_per_task_ms": 316187.70707070705, "avg_output_tokens_per_task": 30484.299663299662, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9027699539584503, "accuracy_std_dev": 0.006295075818094274, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.028251830432341377, "avg_duration_per_task_ms": 172539.25996631107, "avg_output_tokens_per_task": 28841.043234138124, "last_run_timestamp": "2026-10-31T08:31:10.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902357, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1437863712121212, "avg_duration_per_task_ms": 149585.63636363635, "avg_output_tokens_per_task": 9456.126262626263, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.9023568, "accuracy_std_dev": 0.003838970664123373, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0695745484848485, "avg_duration_per_task_ms": 344238.6484848485, "avg_output_tokens_per_task": 16158.315151515151, "last_run_timestamp": "2026-10-31T08:52:05.708Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.9020761666666667, "accuracy_std_dev": 0.005271695534223824, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05407119865319866, "avg_duration_per_task_ms": 30482.292087542086, "avg_output_tokens_per_task": 2495.6936026936028, "last_run_timestamp": "2026-10-31T08:27:47.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "PrimeIntellect", "accuracy": 0.9018877466467959, "accuracy_std_dev": 0.01830165892678099, "run_count": 6, "total_questions": 671, "avg_cost_per_task_usd": 0.0035507228017883754, "avg_duration_per_task_ms": 96273.04172876304, "avg_output_tokens_per_task": 15221.761549925484, "last_run_timestamp": "2026-11-02T17:22:52.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.9015150000000001, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0736977297979798, "avg_duration_per_task_ms": 138589.0126262626, "avg_output_tokens_per_task": 17125.45707070707, "last_run_timestamp": "2026-10-31T08:09:26.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0720575808080808, "avg_duration_per_task_ms": 100990.45707070707, "avg_output_tokens_per_task": 16730.29797979798, "last_run_timestamp": "2026-10-31T08:17:04.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.901515, "accuracy_std_dev": 0.004532823209656408, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13843164835858585, "avg_duration_per_task_ms": 1759646.8768939395, "avg_output_tokens_per_task": 58099.75189393939, "last_run_timestamp": "2026-10-31T16:13:09.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.16319313383838385, "avg_duration_per_task_ms": 88415.74242424243, "avg_output_tokens_per_task": 9762.232323232323, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9014423347902099, "accuracy_std_dev": 0.008872458280591035, "run_count": 13, "total_questions": 2288, "avg_cost_per_task_usd": 0.02873364816433567, "avg_duration_per_task_ms": 177751.28103146853, "avg_output_tokens_per_task": 22378.798513986014, "last_run_timestamp": "2026-11-06T06:01:51.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.9012793199999999, "accuracy_std_dev": 0.006151174747769718, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.20707172282828287, "avg_duration_per_task_ms": 98196.02, "avg_output_tokens_per_task": 7866.467070707071, "last_run_timestamp": "2026-10-31T08:46:46.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.9012343333333332, "accuracy_std_dev": 0.0037221826780652403, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03380553956228956, "avg_duration_per_task_ms": 826187.4343434343, "avg_output_tokens_per_task": 38504.42255892256, "last_run_timestamp": "2026-10-31T08:14:20.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9011541428571429, "accuracy_std_dev": 0.004473489479416358, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.053932944444444436, "avg_duration_per_task_ms": 26662.532467532466, "avg_output_tokens_per_task": 2480.9805194805194, "last_run_timestamp": "2026-10-31T08:25:18.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.9011222848914487, "accuracy_std_dev": 0.007337523255215825, "run_count": 12, "total_questions": 2257, "avg_cost_per_task_usd": 0.013226397430217102, "avg_duration_per_task_ms": 143480.65086397872, "avg_output_tokens_per_task": 26118.721754541428, "last_run_timestamp": "2026-10-31T08:07:19.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.9010944999999999, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00798360101010101, "avg_duration_per_task_ms": 317918.4797979798, "avg_output_tokens_per_task": 28057.032828282827, "last_run_timestamp": "2026-10-31T16:08:33.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.9009462380396732, "accuracy_std_dev": 0.019684329717769646, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.12647589887203423, "avg_duration_per_task_ms": 518480.37456242705, "avg_output_tokens_per_task": 38463.91637495138, "last_run_timestamp": "2026-10-31T08:17:28.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.9006730000000001, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010972851010101009, "avg_duration_per_task_ms": 117479.04797979798, "avg_output_tokens_per_task": 19415.13888888889, "last_run_timestamp": "2026-10-31T08:11:46.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.9005650372881355, "accuracy_std_dev": 0.0005644881654108368, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.013123520338983051, "avg_duration_per_task_ms": 181977.62711864407, "avg_output_tokens_per_task": 21777.491525423728, "last_run_timestamp": "2026-10-31T08:31:11.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.9005043975240714, "accuracy_std_dev": 0.00897585252218416, "run_count": 4, "total_questions": 727, "avg_cost_per_task_usd": 0.027938022008253093, "avg_duration_per_task_ms": 73305.65749656121, "avg_output_tokens_per_task": 18908.090784044016, "last_run_timestamp": "2026-11-05T21:01:47.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9002637837837837, "accuracy_std_dev": 0.005876451733850735, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.2161343588588589, "avg_duration_per_task_ms": 93366.365001365, "avg_output_tokens_per_task": 7877.386704886705, "last_run_timestamp": "2026-10-31T08:23:44.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Nebius", "accuracy": 0.9002525, "accuracy_std_dev": 0.004014983343676526, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014062290404040403, "avg_duration_per_task_ms": 87702.04797979798, "avg_output_tokens_per_task": 13895.555555555555, "last_run_timestamp": "2026-10-31T08:28:22.736Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9000319047619048, "accuracy_std_dev": 0.005076517459315892, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.05824434295334296, "avg_duration_per_task_ms": 35780.01707551708, "avg_output_tokens_per_task": 2517.5358345358345, "last_run_timestamp": "2026-10-31T09:01:04.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.8998314999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11599434848484849, "avg_duration_per_task_ms": 1589231.9873737374, "avg_output_tokens_per_task": 46211.58585858586, "last_run_timestamp": "2026-10-31T08:31:10.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.8996625716694773, "accuracy_std_dev": 0.0020759481367959394, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.11324433389544686, "avg_duration_per_task_ms": 858665.3389544688, "avg_output_tokens_per_task": 45077.612141652615, "last_run_timestamp": "2026-10-31T08:28:18.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8995687234401349, "accuracy_std_dev": 0.007447984730328704, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.0197627712197864, "avg_duration_per_task_ms": 382016.3063518831, "avg_output_tokens_per_task": 22859.64811691962, "last_run_timestamp": "2026-10-31T08:33:02.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8995041666666669, "accuracy_std_dev": 0.005842819508213075, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.22327578970258133, "avg_duration_per_task_ms": 94284.44626823794, "avg_output_tokens_per_task": 8148.210998877666, "last_run_timestamp": "2026-10-31T08:31:12.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8994017354790972, "accuracy_std_dev": 0.12179597930511851, "run_count": 28, "total_questions": 5406, "avg_cost_per_task_usd": 0.16268426008139103, "avg_duration_per_task_ms": 193647.5, "avg_output_tokens_per_task": 10722.489456159823, "last_run_timestamp": "2026-10-31T09:24:16.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8993883798165137, "accuracy_std_dev": 0.03327796223455359, "run_count": 8, "total_questions": 1090, "avg_cost_per_task_usd": 0.02275113119266055, "avg_duration_per_task_ms": 820057.9403669725, "avg_output_tokens_per_task": 21074.469724770643, "last_run_timestamp": "2026-10-31T16:03:21.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8993857647058824, "accuracy_std_dev": 0.018671156495670343, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.07737615121806299, "avg_duration_per_task_ms": 217682.72222222222, "avg_output_tokens_per_task": 30235.886809269163, "last_run_timestamp": "2026-10-31T16:10:09.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8993782307692306, "accuracy_std_dev": 0.014693466141603143, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.004332889665889667, "avg_duration_per_task_ms": 367485.18026418024, "avg_output_tokens_per_task": 23771.594405594406, "last_run_timestamp": "2026-10-31T08:39:08.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8992487692307694, "accuracy_std_dev": 0.016131982491330434, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.020707853146853153, "avg_duration_per_task_ms": 203007.47940947942, "avg_output_tokens_per_task": 26050.9560994561, "last_run_timestamp": "2026-10-31T09:18:59.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8992302857142856, "accuracy_std_dev": 0.0048260090792136764, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.05735562409812409, "avg_duration_per_task_ms": 25146.343674843676, "avg_output_tokens_per_task": 2479.3174603174602, "last_run_timestamp": "2026-10-31T09:06:56.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.8991100000000001, "accuracy_std_dev": 0.005600276243186554, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.05768347799422799, "avg_duration_per_task_ms": 26053.75036075036, "avg_output_tokens_per_task": 2530.0901875901877, "last_run_timestamp": "2026-10-31T08:22:04.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8989896800000002, "accuracy_std_dev": 0.007736742160470359, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.12731747616161618, "avg_duration_per_task_ms": 58520.54424242424, "avg_output_tokens_per_task": 4742.417575757576, "last_run_timestamp": "2026-10-31T08:31:13.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8987081781963057, "accuracy_std_dev": 0.03249433928983099, "run_count": 14, "total_questions": 2761, "avg_cost_per_task_usd": 0.14709861970300617, "avg_duration_per_task_ms": 823480.3976819993, "avg_output_tokens_per_task": 36846.00507062658, "last_run_timestamp": "2026-10-31T08:15:03.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8981480833333332, "accuracy_std_dev": 0.007606843601415013, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19857798400673404, "avg_duration_per_task_ms": 95247.20033670033, "avg_output_tokens_per_task": 7731.943181818182, "last_run_timestamp": "2026-10-31T08:27:45.200Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.8981478333333334, "accuracy_std_dev": 0.007386487717823356, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12543263804713806, "avg_duration_per_task_ms": 965889.989057239, "avg_output_tokens_per_task": 49917.90404040404, "last_run_timestamp": "2026-10-31T08:45:01.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.89814775, "accuracy_std_dev": 0.0055197238778311765, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11915268055555557, "avg_duration_per_task_ms": 825703.6161616162, "avg_output_tokens_per_task": 47450.94823232323, "last_run_timestamp": "2026-10-31T08:07:17.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8980075833333333, "accuracy_std_dev": 0.004784274352123693, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.23032522264309765, "avg_duration_per_task_ms": 110445.48653198653, "avg_output_tokens_per_task": 8998.862373737375, "last_run_timestamp": "2026-10-31T08:24:37.252Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.8978673333333335, "accuracy_std_dev": 0.007570651065499923, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03724634175084175, "avg_duration_per_task_ms": 745311.5892255892, "avg_output_tokens_per_task": 37370.69696969697, "last_run_timestamp": "2026-10-31T08:12:46.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8977871428571428, "accuracy_std_dev": 0.005599042607665047, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15678494083694083, "avg_duration_per_task_ms": 138524.9898989899, "avg_output_tokens_per_task": 9631.916305916306, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8976429, "accuracy_std_dev": 0.019280368027867086, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.024536105555555552, "avg_duration_per_task_ms": 164306.0883838384, "avg_output_tokens_per_task": 28203.131818181817, "last_run_timestamp": "2026-10-31T08:31:11.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8973428130434782, "accuracy_std_dev": 0.010729397445850358, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.19345281014492755, "avg_duration_per_task_ms": 1610739.8637681159, "avg_output_tokens_per_task": 56727.897826086955, "last_run_timestamp": "2026-10-24T16:20:08.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8973063333333333, "accuracy_std_dev": 0.004955863351178606, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03703221212121212, "avg_duration_per_task_ms": 137593.69696969696, "avg_output_tokens_per_task": 26012.949494949495, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.897306, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026144708754208754, "avg_duration_per_task_ms": 176560.02693602693, "avg_output_tokens_per_task": 21638.170033670034, "last_run_timestamp": "2026-10-31T08:57:38.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.8973059999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07477493686868687, "avg_duration_per_task_ms": 142684.398989899, "avg_output_tokens_per_task": 17437.89393939394, "last_run_timestamp": "2026-10-31T08:49:18.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8972595277777778, "accuracy_std_dev": 0.008117361043145963, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.20771573681257013, "avg_duration_per_task_ms": 71213.86826599327, "avg_output_tokens_per_task": 7567.590488215488, "last_run_timestamp": "2026-10-31T08:51:35.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.8970256666666665, "accuracy_std_dev": 0.00667522615513675, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06776180639730639, "avg_duration_per_task_ms": 222686.2255892256, "avg_output_tokens_per_task": 15778.796296296296, "last_run_timestamp": "2026-10-31T08:27:47.368Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8968853333333332, "accuracy_std_dev": 0.018366417657114165, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.032345546296296296, "avg_duration_per_task_ms": 248464.89772727274, "avg_output_tokens_per_task": 27393.215909090908, "last_run_timestamp": "2026-10-31T08:36:32.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8967014333929941, "accuracy_std_dev": 0.03454965314283198, "run_count": 20, "total_questions": 3911, "avg_cost_per_task_usd": 0.37683498056762965, "avg_duration_per_task_ms": 497556.26566095627, "avg_output_tokens_per_task": 57557.85144464331, "last_run_timestamp": "2026-10-31T16:18:08.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.8966514444444446, "accuracy_std_dev": 0.022056702876959412, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.06759996408529742, "avg_duration_per_task_ms": 533319.7861952862, "avg_output_tokens_per_task": 36554.644219977556, "last_run_timestamp": "2026-10-31T16:05:14.632Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8966242936708859, "accuracy_std_dev": 0.010248237855304126, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.017466570042194093, "avg_duration_per_task_ms": 356256.63333333336, "avg_output_tokens_per_task": 28695.1776371308, "last_run_timestamp": "2026-10-31T09:16:30.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8965486500000001, "accuracy_std_dev": 0.014777080321480958, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.007944752525252527, "avg_duration_per_task_ms": 153300.83762626263, "avg_output_tokens_per_task": 23808.83282828283, "last_run_timestamp": "2026-10-31T09:24:59.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.896184, "accuracy_std_dev": 0.007570749896806769, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0237254696969697, "avg_duration_per_task_ms": 319850.3367003367, "avg_output_tokens_per_task": 19569.10606060606, "last_run_timestamp": "2026-10-31T08:20:35.720Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8961839166666666, "accuracy_std_dev": 0.011424969966833842, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.2001774431818182, "avg_duration_per_task_ms": 95856.68686868687, "avg_output_tokens_per_task": 7788.865740740741, "last_run_timestamp": "2026-10-31T08:32:06.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8961037142857144, "accuracy_std_dev": 0.004737329618927418, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.054530834776334774, "avg_duration_per_task_ms": 30476.945887445887, "avg_output_tokens_per_task": 2521.178932178932, "last_run_timestamp": "2026-10-31T08:40:20.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8958817692307695, "accuracy_std_dev": 0.006430888982138585, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.20706321833721833, "avg_duration_per_task_ms": 100110.66355866355, "avg_output_tokens_per_task": 8069.975524475524, "last_run_timestamp": "2026-10-31T08:27:47.283Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.895623, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03841230555555556, "avg_duration_per_task_ms": 144059.0782828283, "avg_output_tokens_per_task": 27039.489898989897, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.895412375, "accuracy_std_dev": 0.020051295631065218, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02414115151515151, "avg_duration_per_task_ms": 210274.94823232322, "avg_output_tokens_per_task": 27810.135101010103, "last_run_timestamp": "2026-10-31T08:27:49.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8952905937499999, "accuracy_std_dev": 0.016969367452854572, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.032397834635416664, "avg_duration_per_task_ms": 236218.794921875, "avg_output_tokens_per_task": 32195.716796875, "last_run_timestamp": "2026-10-31T09:17:17.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.89520175, "accuracy_std_dev": 0.0077796281522640644, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12088184385521886, "avg_duration_per_task_ms": 58635.83712121212, "avg_output_tokens_per_task": 4614.950757575758, "last_run_timestamp": "2026-10-31T08:31:11.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.8951904072144289, "accuracy_std_dev": 0.019820245214898276, "run_count": 13, "total_questions": 2495, "avg_cost_per_task_usd": 0.159686151503006, "avg_duration_per_task_ms": 368440.1851703407, "avg_output_tokens_per_task": 40060.66533066132, "last_run_timestamp": "2026-10-31T08:09:30.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8951417142857143, "accuracy_std_dev": 0.009679824699036726, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008688588023088023, "avg_duration_per_task_ms": 171107.75108225108, "avg_output_tokens_per_task": 30552.396103896102, "last_run_timestamp": "2026-10-31T09:09:17.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8950615555555554, "accuracy_std_dev": 0.010765037370127736, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005121220538720539, "avg_duration_per_task_ms": 221349.66778900113, "avg_output_tokens_per_task": 28186.213804713803, "last_run_timestamp": "2026-10-31T16:31:13.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8950615000000002, "accuracy_std_dev": 0.009654758735980914, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.2321890096801347, "avg_duration_per_task_ms": 110673.35858585859, "avg_output_tokens_per_task": 9063.56186868687, "last_run_timestamp": "2026-10-31T08:13:10.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.894781, "accuracy_std_dev": 0.028308178049814503, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1154155827020202, "avg_duration_per_task_ms": 425629.0599747475, "avg_output_tokens_per_task": 36939.916666666664, "last_run_timestamp": "2026-10-31T16:10:36.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8943087542213883, "accuracy_std_dev": 0.03942204850926279, "run_count": 3, "total_questions": 533, "avg_cost_per_task_usd": 0.019450917448405252, "avg_duration_per_task_ms": 177927.2420262664, "avg_output_tokens_per_task": 28174.943714821762, "last_run_timestamp": "2026-10-28T05:00:14.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8942835463539759, "accuracy_std_dev": 0.01749872529716121, "run_count": 14, "total_questions": 2729, "avg_cost_per_task_usd": 0.1169677999267131, "avg_duration_per_task_ms": 555475.2238915353, "avg_output_tokens_per_task": 43652.794796628805, "last_run_timestamp": "2026-11-06T22:44:36.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.894276, "accuracy_std_dev": 0.008869991905295046, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11497575656565656, "avg_duration_per_task_ms": 654340.1272727273, "avg_output_tokens_per_task": 45820.55656565657, "last_run_timestamp": "2026-10-31T08:45:35.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8941497500000002, "accuracy_std_dev": 0.012326495474687834, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010706986111111112, "avg_duration_per_task_ms": 265829.2607323232, "avg_output_tokens_per_task": 20748.16351010101, "last_run_timestamp": "2026-10-31T08:33:46.211Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "CoreWeave", "accuracy": 0.893939358490566, "accuracy_std_dev": 0.004245539018258487, "run_count": 3, "total_questions": 583, "avg_cost_per_task_usd": 0.023210332761578047, "avg_duration_per_task_ms": 116069.80960548884, "avg_output_tokens_per_task": 13628.193825042881, "last_run_timestamp": "2026-10-31T08:17:29.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.8939393333333333, "accuracy_std_dev": 0.012217692780917704, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02448665488215488, "avg_duration_per_task_ms": 139886.7744107744, "avg_output_tokens_per_task": 20249.15824915825, "last_run_timestamp": "2026-10-31T09:02:19.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.8934709999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 194, "avg_cost_per_task_usd": 0.11686864948453608, "avg_duration_per_task_ms": 605687.7474226804, "avg_output_tokens_per_task": 46496.701030927834, "last_run_timestamp": "2026-10-13T03:00:07.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "BaseTen", "accuracy": 0.8933780000000001, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03135734848484849, "avg_duration_per_task_ms": 31339.205387205388, "avg_output_tokens_per_task": 13928.784511784512, "last_run_timestamp": "2026-10-31T08:28:17.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "DigitalOcean", "accuracy": 0.8933779999999999, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01648433501683502, "avg_duration_per_task_ms": 93266.93602693603, "avg_output_tokens_per_task": 13659.838383838383, "last_run_timestamp": "2026-10-31T08:40:23.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8931983140809443, "accuracy_std_dev": 0.01619141679512122, "run_count": 12, "total_questions": 2372, "avg_cost_per_task_usd": 0.1531594173693086, "avg_duration_per_task_ms": 667713.1500843171, "avg_output_tokens_per_task": 37442.99325463744, "last_run_timestamp": "2026-11-05T00:00:09.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "PrimeIntellect", "accuracy": 0.8931621743589744, "accuracy_std_dev": 0.0023230000000000195, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.020242015384615383, "avg_duration_per_task_ms": 141496.24102564104, "avg_output_tokens_per_task": 40323.36666666667, "last_run_timestamp": "2026-10-31T08:52:13.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8930975, "accuracy_std_dev": 0.010942499999999966, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04334855050505051, "avg_duration_per_task_ms": 40520.42929292929, "avg_output_tokens_per_task": 3151.881313131313, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.8930975, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11303334343434343, "avg_duration_per_task_ms": 814053.6843434343, "avg_output_tokens_per_task": 45048.43686868687, "last_run_timestamp": "2026-10-31T08:27:45.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.8928929372738239, "accuracy_std_dev": 0.015585617718297187, "run_count": 17, "total_questions": 3316, "avg_cost_per_task_usd": 0.014895325693606755, "avg_duration_per_task_ms": 187175.30186972255, "avg_output_tokens_per_task": 21447.96381182147, "last_run_timestamp": "2026-10-31T08:57:07.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8928570861678005, "accuracy_std_dev": 0.015340394005370288, "run_count": 5, "total_questions": 882, "avg_cost_per_task_usd": 0.12683532766439912, "avg_duration_per_task_ms": 285532.1031746032, "avg_output_tokens_per_task": 28366.77097505669, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.892817, "accuracy_std_dev": 0.00915257617650169, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007166784511784513, "avg_duration_per_task_ms": 121588.36531986532, "avg_output_tokens_per_task": 11764.878787878788, "last_run_timestamp": "2026-10-31T08:24:44.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.892817, "accuracy_std_dev": 0.0067805750493597225, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11382627104377104, "avg_duration_per_task_ms": 836209.1372053872, "avg_output_tokens_per_task": 45359.85101010101, "last_run_timestamp": "2026-10-31T09:36:23.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8928169166666666, "accuracy_std_dev": 0.00928060290748336, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.2432147763047138, "avg_duration_per_task_ms": 85093.25778619529, "avg_output_tokens_per_task": 9036.06712962963, "last_run_timestamp": "2026-10-31T08:12:47.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8926684364569961, "accuracy_std_dev": 0.021409854620919034, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.01926024176294395, "avg_duration_per_task_ms": 249786.02053915276, "avg_output_tokens_per_task": 30225.52802738554, "last_run_timestamp": "2026-11-04T21:06:11.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8925062583961616, "accuracy_std_dev": 0.02075156285341843, "run_count": 8, "total_questions": 1459, "avg_cost_per_task_usd": 0.1553590150788211, "avg_duration_per_task_ms": 645745.0089102124, "avg_output_tokens_per_task": 50031.70390678547, "last_run_timestamp": "2026-10-31T16:34:03.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8925003799071338, "accuracy_std_dev": 0.009652319598878228, "run_count": 12, "total_questions": 2369, "avg_cost_per_task_usd": 0.007965678767412412, "avg_duration_per_task_ms": 323776.0540312368, "avg_output_tokens_per_task": 28074.01392992824, "last_run_timestamp": "2026-10-31T08:16:11.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8922560000000002, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14220060606060608, "avg_duration_per_task_ms": 188559.2904040404, "avg_output_tokens_per_task": 47276.156565656565, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8917748571428571, "accuracy_std_dev": 0.009721593908534165, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04607931240981241, "avg_duration_per_task_ms": 43786.40764790765, "avg_output_tokens_per_task": 3453.090909090909, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.8916946666666665, "accuracy_std_dev": 0.009944030347678707, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.011122356902356903, "avg_duration_per_task_ms": 302424.50673400675, "avg_output_tokens_per_task": 25444.905723905726, "last_run_timestamp": "2026-10-31T09:01:06.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8913493076923077, "accuracy_std_dev": 0.011970151958322219, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.20830912587412587, "avg_duration_per_task_ms": 99479.31274281275, "avg_output_tokens_per_task": 8103.869463869464, "last_run_timestamp": "2026-10-31T08:39:17.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8912736666666665, "accuracy_std_dev": 0.018072691158823626, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03149746801346801, "avg_duration_per_task_ms": 189533.00841750842, "avg_output_tokens_per_task": 25443.46254208754, "last_run_timestamp": "2026-10-31T08:53:51.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8910890642064206, "accuracy_std_dev": 0.023037670395819873, "run_count": 17, "total_questions": 3333, "avg_cost_per_task_usd": 0.007215581758175818, "avg_duration_per_task_ms": 268643.4326432643, "avg_output_tokens_per_task": 27966.94419441944, "last_run_timestamp": "2026-11-02T17:25:00.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8910866666666668, "accuracy_std_dev": 0.009472170005395327, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.1333184758698092, "avg_duration_per_task_ms": 59365.143237934906, "avg_output_tokens_per_task": 4764.966189674523, "last_run_timestamp": "2026-10-31T08:46:09.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8910532857142858, "accuracy_std_dev": 0.008331061289180467, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02299151948051948, "avg_duration_per_task_ms": 246585.4624819625, "avg_output_tokens_per_task": 30738.57647907648, "last_run_timestamp": "2026-10-31T08:09:30.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8899710714285712, "accuracy_std_dev": 0.03484950356372031, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.11476333874458873, "avg_duration_per_task_ms": 499182.5613275613, "avg_output_tokens_per_task": 36700.38997113997, "last_run_timestamp": "2026-10-31T08:09:42.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.889876224111283, "accuracy_std_dev": 0.005998909101508626, "run_count": 7, "total_questions": 1294, "avg_cost_per_task_usd": 0.030587522411128287, "avg_duration_per_task_ms": 772887.7256568779, "avg_output_tokens_per_task": 36034.0115919629, "last_run_timestamp": "2026-10-31T08:09:29.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8897900085877862, "accuracy_std_dev": 0.006183862216824232, "run_count": 6, "total_questions": 1048, "avg_cost_per_task_usd": 0.01576835591603053, "avg_duration_per_task_ms": 219138.39885496182, "avg_output_tokens_per_task": 25718.326335877864, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8897305833333333, "accuracy_std_dev": 0.008597912444874173, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.20647175841750842, "avg_duration_per_task_ms": 99405.93939393939, "avg_output_tokens_per_task": 8030.848063973064, "last_run_timestamp": "2026-10-31T09:23:04.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8895201249999999, "accuracy_std_dev": 0.013120057749849085, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10712794823232324, "avg_duration_per_task_ms": 473198.4646464646, "avg_output_tokens_per_task": 37189.18434343435, "last_run_timestamp": "2026-10-31T08:45:16.520Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8891708512988952, "accuracy_std_dev": 0.008107170901595804, "run_count": 17, "total_questions": 3349, "avg_cost_per_task_usd": 0.015164630934607346, "avg_duration_per_task_ms": 200146.0895789788, "avg_output_tokens_per_task": 27111.550313526426, "last_run_timestamp": "2026-10-31T09:08:32.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.8888890000000002, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0431288872053872, "avg_duration_per_task_ms": 52519.67676767677, "avg_output_tokens_per_task": 3463.8030303030305, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.888889, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1979060606060606, "avg_duration_per_task_ms": 95702.44696969698, "avg_output_tokens_per_task": 7691.454545454545, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Nebius", "accuracy": 0.888889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13065590404040406, "avg_duration_per_task_ms": 663129.7929292929, "avg_output_tokens_per_task": 36983.707070707074, "last_run_timestamp": "2026-11-03T21:00:06.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.8881691813471503, "accuracy_std_dev": 0.0030120000000000147, "run_count": 2, "total_questions": 386, "avg_cost_per_task_usd": 0.1504010103626943, "avg_duration_per_task_ms": 651195.4274611399, "avg_output_tokens_per_task": 42839.41709844559, "last_run_timestamp": "2026-10-31T16:08:08.539Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8876718896377422, "accuracy_std_dev": 0.0038308239938977078, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.032938989890480205, "avg_duration_per_task_ms": 750389.2257792755, "avg_output_tokens_per_task": 38026.47177759057, "last_run_timestamp": "2026-10-31T08:03:09.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.887631592274678, "accuracy_std_dev": 0.04755846530025938, "run_count": 13, "total_questions": 2563, "avg_cost_per_task_usd": 0.07255826414358174, "avg_duration_per_task_ms": 586062.0109246976, "avg_output_tokens_per_task": 39208.97737026922, "last_run_timestamp": "2026-10-31T16:51:33.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.8875500348763478, "accuracy_std_dev": 0.013802209238356553, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.1294895707038681, "avg_duration_per_task_ms": 1720933.2999365884, "avg_output_tokens_per_task": 32195.95688015219, "last_run_timestamp": "2026-10-31T16:35:08.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8874858333333334, "accuracy_std_dev": 0.011999648139517349, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.028057361531986534, "avg_duration_per_task_ms": 346451.00589225587, "avg_output_tokens_per_task": 27164.929713804715, "last_run_timestamp": "2026-10-31T09:11:16.308Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8872051428571429, "accuracy_std_dev": 0.02235225083858491, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.23087303607503606, "avg_duration_per_task_ms": 953449.9184704184, "avg_output_tokens_per_task": 57894.41414141414, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8871015424486148, "accuracy_std_dev": 0.007433711649565692, "run_count": 6, "total_questions": 1119, "avg_cost_per_task_usd": 0.100128890974084, "avg_duration_per_task_ms": 379563.4110813226, "avg_output_tokens_per_task": 36832.25379803396, "last_run_timestamp": "2026-10-31T16:50:08.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8870650833333332, "accuracy_std_dev": 0.018422131673878617, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.013124187710437711, "avg_duration_per_task_ms": 345264.73526936024, "avg_output_tokens_per_task": 23251.8265993266, "last_run_timestamp": "2026-10-31T08:24:31.558Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.887042338150289, "accuracy_std_dev": 0.010371066166434932, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.010374203757225434, "avg_duration_per_task_ms": 78284.89884393064, "avg_output_tokens_per_task": 7947.398121387283, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.8865318000000001, "accuracy_std_dev": 0.007110736653821441, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.030907244444444443, "avg_duration_per_task_ms": 199105.52626262626, "avg_output_tokens_per_task": 25534.58484848485, "last_run_timestamp": "2026-10-31T08:28:17.984Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22157512626262627, "avg_duration_per_task_ms": 106335.13383838384, "avg_output_tokens_per_task": 8638.217171717171, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8863634999999997, "accuracy_std_dev": 0.014636111679108384, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11741310185185187, "avg_duration_per_task_ms": 565931.4103535353, "avg_output_tokens_per_task": 30650.19318181818, "last_run_timestamp": "2026-10-31T16:23:41.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8863451596434361, "accuracy_std_dev": 0.0171093277870711, "run_count": 13, "total_questions": 2468, "avg_cost_per_task_usd": 0.035033690842787686, "avg_duration_per_task_ms": 334174.35372771474, "avg_output_tokens_per_task": 27531.184764991896, "last_run_timestamp": "2026-10-31T08:10:35.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.886153, "accuracy_std_dev": 0.029061101979278085, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.20035622727272726, "avg_duration_per_task_ms": 1000635.4015151515, "avg_output_tokens_per_task": 59277.57196969697, "last_run_timestamp": "2026-10-31T16:24:42.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.886083, "accuracy_std_dev": 0.012320026406898109, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.022550329966329965, "avg_duration_per_task_ms": 149365.32154882155, "avg_output_tokens_per_task": 44803.96127946128, "last_run_timestamp": "2026-10-31T08:13:10.777Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8860398461538462, "accuracy_std_dev": 0.032945951927935986, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.08817068492618493, "avg_duration_per_task_ms": 564559.2735042735, "avg_output_tokens_per_task": 40601.23853923854, "last_run_timestamp": "2026-10-31T16:16:10.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8857262179115301, "accuracy_std_dev": 0.016856765875324656, "run_count": 14, "total_questions": 2758, "avg_cost_per_task_usd": 0.09745979006526469, "avg_duration_per_task_ms": 455192.95939086296, "avg_output_tokens_per_task": 33925.17730239304, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8856471629778674, "accuracy_std_dev": 0.009013068317726206, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.010023977867203217, "avg_duration_per_task_ms": 270739.96177062375, "avg_output_tokens_per_task": 28220.264922870556, "last_run_timestamp": "2026-10-24T09:01:43.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Together", "accuracy": 0.8855216666666669, "accuracy_std_dev": 0.0023805928299947096, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.016851747474747474, "avg_duration_per_task_ms": 62951.132996632994, "avg_output_tokens_per_task": 13874.079124579124, "last_run_timestamp": "2026-10-31T08:03:10.178Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.885161357171555, "accuracy_std_dev": 0.008788568152862834, "run_count": 13, "total_questions": 2489, "avg_cost_per_task_usd": 0.09624794174367216, "avg_duration_per_task_ms": 466564.90518280433, "avg_output_tokens_per_task": 28784.21052631579, "last_run_timestamp": "2026-10-31T16:12:28.231Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8851303585776956, "accuracy_std_dev": 0.013660790573422498, "run_count": 22, "total_questions": 4331, "avg_cost_per_task_usd": 0.1910637587162318, "avg_duration_per_task_ms": 367733.6695913184, "avg_output_tokens_per_task": 31574.25467559455, "last_run_timestamp": "2026-10-24T16:13:46.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8850689656218401, "accuracy_std_dev": 0.014411932672615443, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.026149528816986855, "avg_duration_per_task_ms": 355797.47320525785, "avg_output_tokens_per_task": 26995.41152679474, "last_run_timestamp": "2026-10-10T08:22:45.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.88468, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1319599772727273, "avg_duration_per_task_ms": 249775.19949494948, "avg_output_tokens_per_task": 43863.606060606064, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.8846799999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.08896683838383838, "avg_duration_per_task_ms": 416558.4267676768, "avg_output_tokens_per_task": 22204.20202020202, "last_run_timestamp": "2026-10-31T08:57:00.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8844695, "accuracy_std_dev": 0.019793231336747402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008258296717171718, "avg_duration_per_task_ms": 405954.99242424243, "avg_output_tokens_per_task": 29077.366161616163, "last_run_timestamp": "2026-10-31T16:48:29.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8843993333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004449752525252526, "avg_duration_per_task_ms": 221818.0185185185, "avg_output_tokens_per_task": 21318.58754208754, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.8842879351276745, "accuracy_std_dev": 0.03053132246153121, "run_count": 8, "total_questions": 1449, "avg_cost_per_task_usd": 0.20776448999309868, "avg_duration_per_task_ms": 1449823.9765355417, "avg_output_tokens_per_task": 51794.29537612147, "last_run_timestamp": "2026-10-31T16:04:29.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8842513271500845, "accuracy_std_dev": 0.03154031064539513, "run_count": 18, "total_questions": 3558, "avg_cost_per_task_usd": 0.1480000303541315, "avg_duration_per_task_ms": 355671.73243395163, "avg_output_tokens_per_task": 11013.858909499719, "last_run_timestamp": "2026-10-31T09:14:35.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.8840456666666667, "accuracy_std_dev": 0.005144110505120299, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.02067068717948718, "avg_duration_per_task_ms": 91684.32991452991, "avg_output_tokens_per_task": 41136.99487179487, "last_run_timestamp": "2026-10-31T08:35:13.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8839239383667181, "accuracy_std_dev": 0.011423232412500406, "run_count": 8, "total_questions": 1298, "avg_cost_per_task_usd": 0.00878981278890601, "avg_duration_per_task_ms": 844366.6594761171, "avg_output_tokens_per_task": 29702.687981510015, "last_run_timestamp": "2026-10-31T08:34:49.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04134767424242424, "avg_duration_per_task_ms": 54936.305555555555, "avg_output_tokens_per_task": 3316.282828282828, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014708931818181817, "avg_duration_per_task_ms": 71351.09343434343, "avg_output_tokens_per_task": 13550.636363636364, "last_run_timestamp": "2026-10-31T08:27:52.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "StreamLake", "accuracy": 0.8836312915601023, "accuracy_std_dev": 0.003620000000000012, "run_count": 2, "total_questions": 391, "avg_cost_per_task_usd": 0.006277416879795397, "avg_duration_per_task_ms": 104811.05115089515, "avg_output_tokens_per_task": 10905.734015345268, "last_run_timestamp": "2026-10-31T08:31:24.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8833011934235977, "accuracy_std_dev": 0.027231369868529694, "run_count": 6, "total_questions": 1034, "avg_cost_per_task_usd": 0.09606284816247583, "avg_duration_per_task_ms": 191330.07156673115, "avg_output_tokens_per_task": 25799.25918762089, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8831221012658229, "accuracy_std_dev": 0.01342904913771097, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.1860454797468354, "avg_duration_per_task_ms": 320097.15611814347, "avg_output_tokens_per_task": 46697.94894514768, "last_run_timestamp": "2026-10-31T08:25:00.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8830720146096483, "accuracy_std_dev": 0.009833326482613501, "run_count": 34, "total_questions": 6571, "avg_cost_per_task_usd": 0.14151279226906105, "avg_duration_per_task_ms": 62922.72804748136, "avg_output_tokens_per_task": 5091.605843859382, "last_run_timestamp": "2026-10-31T08:31:11.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.8829965, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006819977272727273, "avg_duration_per_task_ms": 114306.47727272728, "avg_output_tokens_per_task": 11182.35606060606, "last_run_timestamp": "2026-10-31T08:07:19.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.8829290000000001, "accuracy_std_dev": 0.012699110785799135, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.027103435504469985, "avg_duration_per_task_ms": 200899.3601532567, "avg_output_tokens_per_task": 30539.469987228607, "last_run_timestamp": "2026-10-31T08:15:03.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8825948758169935, "accuracy_std_dev": 0.010166708210464918, "run_count": 14, "total_questions": 2754, "avg_cost_per_task_usd": 0.007344207697893974, "avg_duration_per_task_ms": 375236.5925925926, "avg_output_tokens_per_task": 20279.856209150326, "last_run_timestamp": "2026-10-31T08:07:18.580Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.882365125, "accuracy_std_dev": 0.0043279540904883765, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.029606650252525252, "avg_duration_per_task_ms": 78746.81123737374, "avg_output_tokens_per_task": 24519.371843434343, "last_run_timestamp": "2026-10-31T09:02:59.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8821789256756756, "accuracy_std_dev": 0.01706691482241463, "run_count": 6, "total_questions": 1184, "avg_cost_per_task_usd": 0.04157157263513513, "avg_duration_per_task_ms": 164106.9873310811, "avg_output_tokens_per_task": 34476.61317567567, "last_run_timestamp": "2026-10-31T08:31:11.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.8821550000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026309151515151515, "avg_duration_per_task_ms": 202247.18686868687, "avg_output_tokens_per_task": 39119.5505050505, "last_run_timestamp": "2026-10-29T03:00:03.996Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.8821550000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.129083601010101, "avg_duration_per_task_ms": 388751.6590909091, "avg_output_tokens_per_task": 32112.176767676767, "last_run_timestamp": "2026-10-17T16:07:32.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.88215475, "accuracy_std_dev": 0.013094556929789563, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007754531565656565, "avg_duration_per_task_ms": 220861.55050505052, "avg_output_tokens_per_task": 27364.97095959596, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.881818, "accuracy_std_dev": 0.03925680632450885, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010723755555555554, "avg_duration_per_task_ms": 145330.13333333333, "avg_output_tokens_per_task": 16217.381818181819, "last_run_timestamp": "2026-10-31T09:01:13.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8817339999999999, "accuracy_std_dev": 0.016681744617994862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008588069444444443, "avg_duration_per_task_ms": 150888.15656565657, "avg_output_tokens_per_task": 16172.003787878788, "last_run_timestamp": "2026-10-31T08:07:19.698Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.8810325, "accuracy_std_dev": 0.016020429392393534, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03953295538720539, "avg_duration_per_task_ms": 203574.52356902356, "avg_output_tokens_per_task": 32800.2398989899, "last_run_timestamp": "2026-10-31T08:47:44.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8809202687667559, "accuracy_std_dev": 0.012345881812749338, "run_count": 8, "total_questions": 1492, "avg_cost_per_task_usd": 0.0663811581769437, "avg_duration_per_task_ms": 315790.1621983914, "avg_output_tokens_per_task": 18342.8572386059, "last_run_timestamp": "2026-10-31T16:03:21.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8808921666666667, "accuracy_std_dev": 0.019159369526132352, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12913603156565656, "avg_duration_per_task_ms": 63290.436026936026, "avg_output_tokens_per_task": 4931.569023569024, "last_run_timestamp": "2026-10-31T08:31:10.506Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8805721763140385, "accuracy_std_dev": 0.01575367618208524, "run_count": 8, "total_questions": 1503, "avg_cost_per_task_usd": 0.18305890685296072, "avg_duration_per_task_ms": 788943.5455755156, "avg_output_tokens_per_task": 52764.211576846305, "last_run_timestamp": "2026-10-31T16:35:50.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8804710769230769, "accuracy_std_dev": 0.0116074195466755, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.006037170163170164, "avg_duration_per_task_ms": 408315.95765345765, "avg_output_tokens_per_task": 30238.239316239316, "last_run_timestamp": "2026-10-31T08:19:56.454Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8800631670886074, "accuracy_std_dev": 0.006743341790240229, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.033778830379746834, "avg_duration_per_task_ms": 152045.89430379748, "avg_output_tokens_per_task": 27961.480379746834, "last_run_timestamp": "2026-10-31T09:42:46.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8799569142550911, "accuracy_std_dev": 0.006792606115475872, "run_count": 5, "total_questions": 933, "avg_cost_per_task_usd": 0.11584198606645231, "avg_duration_per_task_ms": 484616.05144694535, "avg_output_tokens_per_task": 33104.76848874598, "last_run_timestamp": "2026-10-17T16:07:16.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.879872690530519, "accuracy_std_dev": 0.053065810239372686, "run_count": 18, "total_questions": 3506, "avg_cost_per_task_usd": 0.00976165202509983, "avg_duration_per_task_ms": 414485.4175698802, "avg_output_tokens_per_task": 26831.222760981174, "last_run_timestamp": "2026-10-31T08:47:06.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.87984, "accuracy_std_dev": 0.018052164925570553, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.036141941919191914, "avg_duration_per_task_ms": 139656.41224747474, "avg_output_tokens_per_task": 28081.721590909092, "last_run_timestamp": "2026-10-31T09:14:35.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8793139256044638, "accuracy_std_dev": 0.0299771488382257, "run_count": 9, "total_questions": 1613, "avg_cost_per_task_usd": 0.03197218970861749, "avg_duration_per_task_ms": 352833.73217606946, "avg_output_tokens_per_task": 26068.50278983261, "last_run_timestamp": "2026-10-31T16:15:26.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8793102638453499, "accuracy_std_dev": 0.020773557275777312, "run_count": 10, "total_questions": 1914, "avg_cost_per_task_usd": 0.09333590334378267, "avg_duration_per_task_ms": 520471.578892372, "avg_output_tokens_per_task": 28040.05590386625, "last_run_timestamp": "2026-10-31T08:31:12.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8792495990621337, "accuracy_std_dev": 0.015176521845450746, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.10830853575615476, "avg_duration_per_task_ms": 503979.6647127784, "avg_output_tokens_per_task": 31069.37202032044, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8790466153846153, "accuracy_std_dev": 0.013384300197438356, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.008804418026418025, "avg_duration_per_task_ms": 213500.61344211345, "avg_output_tokens_per_task": 31022.890054390053, "last_run_timestamp": "2026-10-31T08:31:11.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8790284807947019, "accuracy_std_dev": 0.024140287647157404, "run_count": 4, "total_questions": 755, "avg_cost_per_task_usd": 0.004138932450331125, "avg_duration_per_task_ms": 381646.0185430464, "avg_output_tokens_per_task": 15960.21986754967, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8789280833333333, "accuracy_std_dev": 0.018215839684453818, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12799191666666665, "avg_duration_per_task_ms": 62532.678872053875, "avg_output_tokens_per_task": 4895.166245791246, "last_run_timestamp": "2026-10-31T08:12:45.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8787880000000001, "accuracy_std_dev": 0.007273551310513108, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009556249158249158, "avg_duration_per_task_ms": 52070.99494949495, "avg_output_tokens_per_task": 7331.292929292929, "last_run_timestamp": "2026-10-30T00:00:06.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.878787777777778, "accuracy_std_dev": 0.016872358550651074, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.023912988215488217, "avg_duration_per_task_ms": 730084.7261503928, "avg_output_tokens_per_task": 36009.7519640853, "last_run_timestamp": "2026-10-31T16:46:00.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8787235883100379, "accuracy_std_dev": 0.03310670771944613, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.10775736086404068, "avg_duration_per_task_ms": 649248.8576874206, "avg_output_tokens_per_task": 26784.826768318508, "last_run_timestamp": "2026-10-31T16:08:08.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.878634635903919, "accuracy_std_dev": 0.006072744766577977, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.11808843784239359, "avg_duration_per_task_ms": 774332.4475347662, "avg_output_tokens_per_task": 30907.657395701644, "last_run_timestamp": "2026-10-31T16:19:28.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.8782263333333332, "accuracy_std_dev": 0.004199386807089298, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020176612794612796, "avg_duration_per_task_ms": 120195.73063973064, "avg_output_tokens_per_task": 42971.59090909091, "last_run_timestamp": "2026-10-31T08:09:30.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8781846784586228, "accuracy_std_dev": 0.00428175396303899, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.00286223562855338, "avg_duration_per_task_ms": 91558.78395451674, "avg_output_tokens_per_task": 5494.553379658875, "last_run_timestamp": "2026-11-03T03:00:05.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.8774407999999999, "accuracy_std_dev": 0.00954717737134911, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03921924343434343, "avg_duration_per_task_ms": 242470.04242424242, "avg_output_tokens_per_task": 37354.009090909094, "last_run_timestamp": "2026-10-21T11:23:15.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8773146249999999, "accuracy_std_dev": 0.019550283047423515, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.20197871338383838, "avg_duration_per_task_ms": 597313.053030303, "avg_output_tokens_per_task": 59103.44823232323, "last_run_timestamp": "2026-10-31T16:09:16.278Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8771411280539175, "accuracy_std_dev": 0.030213255709644882, "run_count": 12, "total_questions": 2374, "avg_cost_per_task_usd": 0.17571395745577084, "avg_duration_per_task_ms": 362774.2093513058, "avg_output_tokens_per_task": 39884.13732097725, "last_run_timestamp": "2026-10-31T16:32:36.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8771043749999999, "accuracy_std_dev": 0.026498402465702973, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13848938320707072, "avg_duration_per_task_ms": 860393.4109848485, "avg_output_tokens_per_task": 62250.43244949495, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.876730111111111, "accuracy_std_dev": 0.012390967395149362, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03705083277216611, "avg_duration_per_task_ms": 153849.39562289562, "avg_output_tokens_per_task": 30722.645342312007, "last_run_timestamp": "2026-11-05T00:00:09.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8767260400808354, "accuracy_std_dev": 0.008082492995494786, "run_count": 15, "total_questions": 2969, "avg_cost_per_task_usd": 0.024447516335466487, "avg_duration_per_task_ms": 220112.43718423712, "avg_output_tokens_per_task": 34286.534523408554, "last_run_timestamp": "2026-11-02T17:21:00.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8765656184971099, "accuracy_std_dev": 0.007664911544146613, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.12350613511560693, "avg_duration_per_task_ms": 893845.7933526011, "avg_output_tokens_per_task": 43364.7362716763, "last_run_timestamp": "2026-10-24T16:19:29.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8763216813793104, "accuracy_std_dev": 0.01096372812565248, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.0099918183908046, "avg_duration_per_task_ms": 425649.0579310345, "avg_output_tokens_per_task": 22880.83264367816, "last_run_timestamp": "2026-10-31T08:32:56.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8762625, "accuracy_std_dev": 0.009259500000000032, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014013469696969696, "avg_duration_per_task_ms": 171317.10353535353, "avg_output_tokens_per_task": 26009.79292929293, "last_run_timestamp": "2026-10-31T08:09:25.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8762625, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009382047979797978, "avg_duration_per_task_ms": 83118.84595959596, "avg_output_tokens_per_task": 7689.015151515152, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.8759833172804533, "accuracy_std_dev": 0.02383166285579381, "run_count": 6, "total_questions": 1059, "avg_cost_per_task_usd": 0.006660402266288952, "avg_duration_per_task_ms": 451840.41265344666, "avg_output_tokens_per_task": 36610.579792256845, "last_run_timestamp": "2026-10-31T08:15:03.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.8759819999999999, "accuracy_std_dev": 0.04446362050785635, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019776964646464647, "avg_duration_per_task_ms": 142117.47643097644, "avg_output_tokens_per_task": 20178.936026936026, "last_run_timestamp": "2026-10-31T09:01:06.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.8755709520547945, "accuracy_std_dev": 0.007533567739715949, "run_count": 3, "total_questions": 584, "avg_cost_per_task_usd": 0.031521625, "avg_duration_per_task_ms": 247947.07705479453, "avg_output_tokens_per_task": 62802.890410958906, "last_run_timestamp": "2026-10-31T08:14:43.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.875421, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05678013888888889, "avg_duration_per_task_ms": 453449.72222222225, "avg_output_tokens_per_task": 62821.7904040404, "last_run_timestamp": "2026-10-31T08:23:57.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.8754206666666668, "accuracy_std_dev": 0.004955863351178644, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01787970538720539, "avg_duration_per_task_ms": 181946.53367003368, "avg_output_tokens_per_task": 29504.771043771045, "last_run_timestamp": "2026-10-31T08:09:31.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04156198232323233, "avg_duration_per_task_ms": 309570.8888888889, "avg_output_tokens_per_task": 39159.9494949495, "last_run_timestamp": "2026-10-17T08:26:18.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.8752804166666668, "accuracy_std_dev": 0.013423450409999739, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.005922073653198653, "avg_duration_per_task_ms": 268671.5, "avg_output_tokens_per_task": 18183.949074074073, "last_run_timestamp": "2026-10-31T08:36:26.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.87521025, "accuracy_std_dev": 0.005483586138422573, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.08232593308080809, "avg_duration_per_task_ms": 423311.8996212121, "avg_output_tokens_per_task": 23409.34974747475, "last_run_timestamp": "2026-10-31T16:15:54.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8749371894736842, "accuracy_std_dev": 0.025253942161324032, "run_count": 4, "total_questions": 665, "avg_cost_per_task_usd": 0.11522267669172932, "avg_duration_per_task_ms": 531594.2751879699, "avg_output_tokens_per_task": 48718.98195488722, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8748596666666667, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.09818961279461279, "avg_duration_per_task_ms": 617620.867003367, "avg_output_tokens_per_task": 28483.28787878788, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.8746087667870037, "accuracy_std_dev": 0.007592482877484693, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.2179415407942238, "avg_duration_per_task_ms": 1239989.4288808664, "avg_output_tokens_per_task": 48485.83321299639, "last_run_timestamp": "2026-10-24T16:18:35.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.874579, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11236396717171716, "avg_duration_per_task_ms": 447332.6186868687, "avg_output_tokens_per_task": 32120.56313131313, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8744105999999999, "accuracy_std_dev": 0.014149500063253112, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09050255303030302, "avg_duration_per_task_ms": 196310.64444444445, "avg_output_tokens_per_task": 14775.43686868687, "last_run_timestamp": "2026-10-31T08:07:27.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8743250351006383, "accuracy_std_dev": 0.026296453097608783, "run_count": 21, "total_questions": 4074, "avg_cost_per_task_usd": 0.03975532817869414, "avg_duration_per_task_ms": 80921.18360333824, "avg_output_tokens_per_task": 25880.549828178693, "last_run_timestamp": "2026-10-31T09:03:20.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.8742983333333334, "accuracy_std_dev": 0.0055553002519115975, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02925406565656566, "avg_duration_per_task_ms": 153006.37373737374, "avg_output_tokens_per_task": 28825.781144781144, "last_run_timestamp": "2026-10-31T08:50:42.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8740494840344654, "accuracy_std_dev": 0.0038017229265163467, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.04434169640141917, "avg_duration_per_task_ms": 202197.7790167258, "avg_output_tokens_per_task": 38209.482513938165, "last_run_timestamp": "2026-10-31T08:10:37.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8740176666666666, "accuracy_std_dev": 0.03308150602510245, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19026189393939397, "avg_duration_per_task_ms": 46363.66582491583, "avg_output_tokens_per_task": 3596.4814814814813, "last_run_timestamp": "2026-10-31T08:26:18.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8739667650531286, "accuracy_std_dev": 0.012755233259575166, "run_count": 9, "total_questions": 1694, "avg_cost_per_task_usd": 0.008466527744982291, "avg_duration_per_task_ms": 226306.17237308147, "avg_output_tokens_per_task": 29408.253837072018, "last_run_timestamp": "2026-10-28T03:34:12.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8737693295358652, "accuracy_std_dev": 0.014113612302391693, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.09247935569620254, "avg_duration_per_task_ms": 203634.65358649788, "avg_output_tokens_per_task": 23372.77805907173, "last_run_timestamp": "2026-10-31T08:40:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.873737, "accuracy_std_dev": 0.007273551310513108, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01830383333333333, "avg_duration_per_task_ms": 381305.08585858584, "avg_output_tokens_per_task": 30305.045454545456, "last_run_timestamp": "2026-10-31T08:27:46.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8735266250000001, "accuracy_std_dev": 0.01725979185808378, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03874042929292929, "avg_duration_per_task_ms": 506055.12563131313, "avg_output_tokens_per_task": 35547.941287878784, "last_run_timestamp": "2026-10-31T16:06:00.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8727552499999999, "accuracy_std_dev": 0.03887536655159452, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.18207171254208754, "avg_duration_per_task_ms": 44211.63804713805, "avg_output_tokens_per_task": 3431.4545454545455, "last_run_timestamp": "2026-10-31T08:13:57.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.872685, "accuracy_std_dev": 0.023807447553234246, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.11981438194444444, "avg_duration_per_task_ms": 617486.5606060605, "avg_output_tokens_per_task": 35339.65404040404, "last_run_timestamp": "2026-10-31T16:25:53.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8726149999999999, "accuracy_std_dev": 0.0020998715833752102, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017651676767676767, "avg_duration_per_task_ms": 95564.90235690236, "avg_output_tokens_per_task": 29293.20202020202, "last_run_timestamp": "2026-10-31T08:33:05.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8725128181818181, "accuracy_std_dev": 0.06935719834676424, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.00775320890725436, "avg_duration_per_task_ms": 564125.6129476584, "avg_output_tokens_per_task": 31933.148301193756, "last_run_timestamp": "2026-10-31T08:07:18.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8723242701325178, "accuracy_std_dev": 0.009812683936620001, "run_count": 10, "total_questions": 1962, "avg_cost_per_task_usd": 0.008813283384301733, "avg_duration_per_task_ms": 356040.2660550459, "avg_output_tokens_per_task": 31140.901121304792, "last_run_timestamp": "2026-10-17T08:23:51.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.8722639999999998, "accuracy_std_dev": 0.01391911622553672, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01943295707070707, "avg_duration_per_task_ms": 314686.08207070705, "avg_output_tokens_per_task": 35123.857954545456, "last_run_timestamp": "2026-10-31T08:09:31.159Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8717947243121842, "accuracy_std_dev": 0.00772939649952788, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.04942068893879843, "avg_duration_per_task_ms": 291097.89219539583, "avg_output_tokens_per_task": 20991.978663672093, "last_run_timestamp": "2026-10-31T16:14:42.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.8717391442028984, "accuracy_std_dev": 0.05316368413010626, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.18806824130434782, "avg_duration_per_task_ms": 1566628.5471014492, "avg_output_tokens_per_task": 54424.17246376812, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.8714925000000001, "accuracy_std_dev": 0.007251768353305299, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007901208754208754, "avg_duration_per_task_ms": 584978.2028619528, "avg_output_tokens_per_task": 44690.53282828283, "last_run_timestamp": "2026-10-31T16:48:05.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8714859264426126, "accuracy_std_dev": 0.00752103815972237, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.06945589727330374, "avg_duration_per_task_ms": 276858.71528218134, "avg_output_tokens_per_task": 11308.629676601142, "last_run_timestamp": "2026-10-10T08:22:14.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.871422375, "accuracy_std_dev": 0.01757503885442008, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0447298547979798, "avg_duration_per_task_ms": 688258.494949495, "avg_output_tokens_per_task": 35524.118055555555, "last_run_timestamp": "2026-10-31T16:16:55.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8712119166666668, "accuracy_std_dev": 0.038220154043075064, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1817093308080808, "avg_duration_per_task_ms": 45516.91035353536, "avg_output_tokens_per_task": 3430.526515151515, "last_run_timestamp": "2026-10-31T08:20:29.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8708191333333333, "accuracy_std_dev": 0.005011630059061523, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.004255378451178452, "avg_duration_per_task_ms": 39486.23198653199, "avg_output_tokens_per_task": 7745.265656565656, "last_run_timestamp": "2026-10-31T08:49:45.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.8703704, "accuracy_std_dev": 0.00583193054142453, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004018268686868687, "avg_duration_per_task_ms": 37454.91717171717, "avg_output_tokens_per_task": 7796.7444444444445, "last_run_timestamp": "2026-10-31T08:14:20.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8703605355249205, "accuracy_std_dev": 0.03387763198941155, "run_count": 10, "total_questions": 1886, "avg_cost_per_task_usd": 0.014264545599151641, "avg_duration_per_task_ms": 348340.7725344645, "avg_output_tokens_per_task": 28651.488865323437, "last_run_timestamp": "2026-10-31T08:28:24.326Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8701296753246752, "accuracy_std_dev": 0.0026430000000000065, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.09188206233766234, "avg_duration_per_task_ms": 599880.7792207792, "avg_output_tokens_per_task": 27189.84935064935, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.870112333932584, "accuracy_std_dev": 0.07630745338957075, "run_count": 23, "total_questions": 4450, "avg_cost_per_task_usd": 0.21229808741573034, "avg_duration_per_task_ms": 491047.8352808989, "avg_output_tokens_per_task": 61310.42764044944, "last_run_timestamp": "2026-10-31T09:02:18.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.8700336, "accuracy_std_dev": 0.008343089106559986, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07480730858585859, "avg_duration_per_task_ms": 106906.41464646465, "avg_output_tokens_per_task": 12161.163636363637, "last_run_timestamp": "2026-10-31T08:03:09.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8699400591341078, "accuracy_std_dev": 0.01681928135474285, "run_count": 10, "total_questions": 1894, "avg_cost_per_task_usd": 0.07171590390707498, "avg_duration_per_task_ms": 408524.5543822598, "avg_output_tokens_per_task": 23628.092397043296, "last_run_timestamp": "2026-10-31T08:45:16.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "PrimaLabs", "accuracy": 0.8697916770833333, "accuracy_std_dev": 0.045953881606091436, "run_count": 3, "total_questions": 576, "avg_cost_per_task_usd": 0.32188484548611107, "avg_duration_per_task_ms": 172479.83506944444, "avg_output_tokens_per_task": 40101.135416666664, "last_run_timestamp": "2026-11-06T16:32:49.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8697338870630417, "accuracy_std_dev": 0.011260986943785035, "run_count": 17, "total_questions": 3347, "avg_cost_per_task_usd": 0.04833613654018524, "avg_duration_per_task_ms": 200583.7881685091, "avg_output_tokens_per_task": 37254.27158649537, "last_run_timestamp": "2026-10-31T08:35:39.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8695517762430939, "accuracy_std_dev": 0.027759107412948937, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.10362995395948434, "avg_duration_per_task_ms": 356439.69521178637, "avg_output_tokens_per_task": 23309.89779005525, "last_run_timestamp": "2026-10-31T08:27:45.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.00624258508151873, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008183505681818182, "avg_duration_per_task_ms": 364666.8503787879, "avg_output_tokens_per_task": 28881.530303030304, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8695284999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2202375707070707, "avg_duration_per_task_ms": 264875.98484848486, "avg_output_tokens_per_task": 21926.78787878788, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8695284999999998, "accuracy_std_dev": 0.01157224020015143, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.018078708333333332, "avg_duration_per_task_ms": 141813.27146464647, "avg_output_tokens_per_task": 29935.99873737374, "last_run_timestamp": "2026-10-31T08:24:37.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8693882500000001, "accuracy_std_dev": 0.037552927930954993, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.18755991456228957, "avg_duration_per_task_ms": 46608.63804713805, "avg_output_tokens_per_task": 3539.8329124579122, "last_run_timestamp": "2026-10-31T08:09:25.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8693602, "accuracy_std_dev": 0.027670849909607036, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.15936079595959599, "avg_duration_per_task_ms": 153687.5486868687, "avg_output_tokens_per_task": 15356.345858585859, "last_run_timestamp": "2026-10-31T08:57:30.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8690236000000001, "accuracy_std_dev": 0.0056741455250989255, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004051724242424242, "avg_duration_per_task_ms": 38666.534343434345, "avg_output_tokens_per_task": 7872.538383838384, "last_run_timestamp": "2026-10-31T08:35:55.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.868965475862069, "accuracy_std_dev": 0.09920724349312471, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.026564598160919543, "avg_duration_per_task_ms": 193083.65563218392, "avg_output_tokens_per_task": 22703.94482758621, "last_run_timestamp": "2026-10-31T16:04:29.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8686372647058822, "accuracy_std_dev": 0.02658550229012359, "run_count": 34, "total_questions": 6732, "avg_cost_per_task_usd": 0.17476259848484849, "avg_duration_per_task_ms": 159604.14185977422, "avg_output_tokens_per_task": 16236.420083184788, "last_run_timestamp": "2026-10-31T08:33:01.888Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8682058571428571, "accuracy_std_dev": 0.007673379706651383, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07688844300144301, "avg_duration_per_task_ms": 83582.13492063493, "avg_output_tokens_per_task": 12574.14357864358, "last_run_timestamp": "2026-10-10T08:21:46.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8681285334935416, "accuracy_std_dev": 0.03709653553275389, "run_count": 17, "total_questions": 3329, "avg_cost_per_task_usd": 0.01566810363472514, "avg_duration_per_task_ms": 433897.7128266747, "avg_output_tokens_per_task": 40485.41213577651, "last_run_timestamp": "2026-11-04T21:50:43.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.8681253333333332, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.021851893939393942, "avg_duration_per_task_ms": 524499.1397306398, "avg_output_tokens_per_task": 46276.513468013465, "last_run_timestamp": "2026-10-31T08:03:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.867634625, "accuracy_std_dev": 0.007431114989311851, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025694838383838382, "avg_duration_per_task_ms": 248942.36553030304, "avg_output_tokens_per_task": 21167.008207070707, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8676153636363637, "accuracy_std_dev": 0.004014278793432891, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0702307419651056, "avg_duration_per_task_ms": 231760.75160697888, "avg_output_tokens_per_task": 11437.757575757576, "last_run_timestamp": "2026-10-31T08:17:29.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8675645333333333, "accuracy_std_dev": 0.00826267276262484, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.004299042424242424, "avg_duration_per_task_ms": 36217.24208754209, "avg_output_tokens_per_task": 7829.657575757576, "last_run_timestamp": "2026-10-31T08:38:41.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8675643333333334, "accuracy_std_dev": 0.0028614651647170092, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009448984848484849, "avg_duration_per_task_ms": 79377.02525252526, "avg_output_tokens_per_task": 7744.79797979798, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.8674240000000001, "accuracy_std_dev": 0.0048169925783625705, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004064253787878788, "avg_duration_per_task_ms": 38624.37373737374, "avg_output_tokens_per_task": 7893.444444444444, "last_run_timestamp": "2026-10-31T08:27:45.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.8671716000000002, "accuracy_std_dev": 0.006408346388890055, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0041782994949494945, "avg_duration_per_task_ms": 35977.948484848486, "avg_output_tokens_per_task": 7724.74494949495, "last_run_timestamp": "2026-10-31T09:34:38.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8670734166666667, "accuracy_std_dev": 0.04494368828036987, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.1912950467171717, "avg_duration_per_task_ms": 41633.452230639734, "avg_output_tokens_per_task": 3438.911616161616, "last_run_timestamp": "2026-10-31T08:27:45.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.867003, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22265377272727274, "avg_duration_per_task_ms": 265427.9646464646, "avg_output_tokens_per_task": 22169.171717171717, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.867003, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06361645454545455, "avg_duration_per_task_ms": 248018.0404040404, "avg_output_tokens_per_task": 21121.30808080808, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8666972727272727, "accuracy_std_dev": 0.01677101145153263, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.10232939393939393, "avg_duration_per_task_ms": 485607.88200183655, "avg_output_tokens_per_task": 27236.82139577594, "last_run_timestamp": "2026-10-31T16:03:21.454Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.866391090909091, "accuracy_std_dev": 0.02614258303804019, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.007744717630853995, "avg_duration_per_task_ms": 429168.4632690542, "avg_output_tokens_per_task": 26301.81772268136, "last_run_timestamp": "2026-10-31T08:17:29.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8663721249999999, "accuracy_std_dev": 0.06286092990768888, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.2573122026515151, "avg_duration_per_task_ms": 1217529.4318181819, "avg_output_tokens_per_task": 57063.51452020202, "last_run_timestamp": "2026-10-31T16:10:09.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8663051040204212, "accuracy_std_dev": 0.009200990020610552, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.07569640906190171, "avg_duration_per_task_ms": 283298.48819400126, "avg_output_tokens_per_task": 20901.779195915762, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8658810000000001, "accuracy_std_dev": 0.04916542176842041, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11255429335016834, "avg_duration_per_task_ms": 478692.68855218857, "avg_output_tokens_per_task": 28354.992845117846, "last_run_timestamp": "2026-10-31T16:01:53.651Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8656255430210325, "accuracy_std_dev": 0.009966828669561601, "run_count": 9, "total_questions": 1569, "avg_cost_per_task_usd": 0.057410206500956025, "avg_duration_per_task_ms": 438181.7106437221, "avg_output_tokens_per_task": 19027.14722753346, "last_run_timestamp": "2026-10-31T16:05:14.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.86560025, "accuracy_std_dev": 0.030176816372078863, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1543242760942761, "avg_duration_per_task_ms": 151975.23611111112, "avg_output_tokens_per_task": 15215.097222222223, "last_run_timestamp": "2026-10-31T08:27:53.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.865530375, "accuracy_std_dev": 0.017904480611969035, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07363626578282828, "avg_duration_per_task_ms": 288838.01957070705, "avg_output_tokens_per_task": 20955.541666666668, "last_run_timestamp": "2026-10-31T16:09:19.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.86532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01495640909090909, "avg_duration_per_task_ms": 120441.0101010101, "avg_output_tokens_per_task": 29795.70707070707, "last_run_timestamp": "2026-10-24T15:00:02.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8652866062099558, "accuracy_std_dev": 0.03145510165404201, "run_count": 11, "total_questions": 2029, "avg_cost_per_task_usd": 0.023189925086249383, "avg_duration_per_task_ms": 458930.39822572697, "avg_output_tokens_per_task": 22064.744701823558, "last_run_timestamp": "2026-10-17T08:18:04.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8649456666666666, "accuracy_std_dev": 0.026980251460816472, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13166756565656565, "avg_duration_per_task_ms": 468303.6307519641, "avg_output_tokens_per_task": 34458.837261503926, "last_run_timestamp": "2026-10-31T16:30:53.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8647586666666667, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005299535353535353, "avg_duration_per_task_ms": 492201.62457912456, "avg_output_tokens_per_task": 29273.70707070707, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.8647586666666667, "accuracy_std_dev": 0.008943734131906087, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05048964983164983, "avg_duration_per_task_ms": 169537.7323232323, "avg_output_tokens_per_task": 41902.75589225589, "last_run_timestamp": "2026-10-31T08:15:03.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.864685303613644, "accuracy_std_dev": 0.016464492614107477, "run_count": 15, "total_questions": 2961, "avg_cost_per_task_usd": 0.08223853157716987, "avg_duration_per_task_ms": 179300.8497129348, "avg_output_tokens_per_task": 24017.497804795676, "last_run_timestamp": "2026-11-04T21:43:38.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8646462, "accuracy_std_dev": 0.03469156072822321, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005541933333333334, "avg_duration_per_task_ms": 433520.95454545453, "avg_output_tokens_per_task": 17519.044444444444, "last_run_timestamp": "2026-11-05T09:00:05.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.8644780000000001, "accuracy_std_dev": 0.014143861990276906, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7500333691077442, "avg_duration_per_task_ms": 503555.07028619526, "avg_output_tokens_per_task": 29837.847643097644, "last_run_timestamp": "2026-10-31T08:23:25.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8642675, "accuracy_std_dev": 0.01795386030217457, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.16045302777777776, "avg_duration_per_task_ms": 1018899.9987373737, "avg_output_tokens_per_task": 50975.2380050505, "last_run_timestamp": "2026-10-31T16:05:14.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8641672550805886, "accuracy_std_dev": 0.016481482149957736, "run_count": 8, "total_questions": 1427, "avg_cost_per_task_usd": 0.00518134197617379, "avg_duration_per_task_ms": 698929.2803083392, "avg_output_tokens_per_task": 28479.621583742115, "last_run_timestamp": "2026-10-31T16:22:14.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8640571666666669, "accuracy_std_dev": 0.010173704494376127, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8425685521885522, "avg_duration_per_task_ms": 410851.17003367003, "avg_output_tokens_per_task": 33511.65193602694, "last_run_timestamp": "2026-10-31T08:13:57.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.863852179419525, "accuracy_std_dev": 0.02754414592685712, "run_count": 10, "total_questions": 1895, "avg_cost_per_task_usd": 0.012207302902374668, "avg_duration_per_task_ms": 440151.53984168865, "avg_output_tokens_per_task": 28225.067018469657, "last_run_timestamp": "2026-10-31T08:36:30.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.8636363333333335, "accuracy_std_dev": 0.008473463964374614, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.056405106902356894, "avg_duration_per_task_ms": 429778.9991582492, "avg_output_tokens_per_task": 55743.61616161616, "last_run_timestamp": "2026-10-31T16:10:09.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8633403579175706, "accuracy_std_dev": 0.03134350405895737, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.03313610050614606, "avg_duration_per_task_ms": 634565.6637744035, "avg_output_tokens_per_task": 38353.89515545915, "last_run_timestamp": "2026-10-31T08:09:30.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8630751666666665, "accuracy_std_dev": 0.009048483766478349, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0646409191919192, "avg_duration_per_task_ms": 213655.1308922559, "avg_output_tokens_per_task": 10505.142255892257, "last_run_timestamp": "2026-10-31T08:13:57.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8629184270372711, "accuracy_std_dev": 0.04098321044018362, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.1642263594440935, "avg_duration_per_task_ms": 968291.7119393557, "avg_output_tokens_per_task": 55352.77511054959, "last_run_timestamp": "2026-10-31T16:11:41.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8627945000000001, "accuracy_std_dev": 0.00840343408474575, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7843768287037037, "avg_duration_per_task_ms": 448345.18644781143, "avg_output_tokens_per_task": 31202.670875420874, "last_run_timestamp": "2026-10-31T08:15:24.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8627944166666667, "accuracy_std_dev": 0.018753065569209103, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7605652819865321, "avg_duration_per_task_ms": 513193.70033670036, "avg_output_tokens_per_task": 30293.478535353537, "last_run_timestamp": "2026-10-31T08:30:25.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8624118181818182, "accuracy_std_dev": 0.010959488075454655, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0702179696969697, "avg_duration_per_task_ms": 231211.1772268136, "avg_output_tokens_per_task": 11433.751147842057, "last_run_timestamp": "2026-10-31T08:01:54.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8622376508158508, "accuracy_std_dev": 0.05418494311168491, "run_count": 11, "total_questions": 2145, "avg_cost_per_task_usd": 0.07540476596736596, "avg_duration_per_task_ms": 485054.5156177156, "avg_output_tokens_per_task": 20286.058275058276, "last_run_timestamp": "2026-10-31T08:40:20.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.8620252278481012, "accuracy_std_dev": 0.030307500000000043, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.02300040253164557, "avg_duration_per_task_ms": 74866.74430379747, "avg_output_tokens_per_task": 18412.721518987342, "last_run_timestamp": "2026-10-17T16:05:00.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8617424166666667, "accuracy_std_dev": 0.0076116939579650746, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.8836652169612793, "avg_duration_per_task_ms": 413694.60353535356, "avg_output_tokens_per_task": 33474.18686868687, "last_run_timestamp": "2026-10-31T08:24:08.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8615641538461539, "accuracy_std_dev": 0.011629020620487162, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.8434282964257965, "avg_duration_per_task_ms": 459445.4759129759, "avg_output_tokens_per_task": 33558.62315462316, "last_run_timestamp": "2026-10-31T08:07:51.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8614316774441879, "accuracy_std_dev": 0.016000304270056075, "run_count": 8, "total_questions": 1299, "avg_cost_per_task_usd": 0.013207368745188604, "avg_duration_per_task_ms": 402522.6505003849, "avg_output_tokens_per_task": 23550.163202463435, "last_run_timestamp": "2026-11-04T21:18:56.861Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8613828571428571, "accuracy_std_dev": 0.010406829149049206, "run_count": 8, "total_questions": 1533, "avg_cost_per_task_usd": 0.004015036529680365, "avg_duration_per_task_ms": 245941.22635355513, "avg_output_tokens_per_task": 28674.964122635356, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8613568027812896, "accuracy_std_dev": 0.010662573677940023, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.0375921409608091, "avg_duration_per_task_ms": 554775.7035398231, "avg_output_tokens_per_task": 34882.013274336285, "last_run_timestamp": "2026-10-31T16:11:17.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8608771111111111, "accuracy_std_dev": 0.03464756893169482, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.16595036826599324, "avg_duration_per_task_ms": 152410.04938271604, "avg_output_tokens_per_task": 15337.304854096521, "last_run_timestamp": "2026-10-31T08:51:08.192Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8607304246575341, "accuracy_std_dev": 0.01148253802030522, "run_count": 7, "total_questions": 1314, "avg_cost_per_task_usd": 0.02474931582952816, "avg_duration_per_task_ms": 139190.8691019787, "avg_output_tokens_per_task": 37895.824200913245, "last_run_timestamp": "2026-10-31T08:41:38.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.860546054237288, "accuracy_std_dev": 0.01522905267434664, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.1431118, "avg_duration_per_task_ms": 301601.77118644066, "avg_output_tokens_per_task": 30086.04463276836, "last_run_timestamp": "2026-10-31T09:06:29.131Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8603986923076923, "accuracy_std_dev": 0.01707241294461563, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.7520348795648795, "avg_duration_per_task_ms": 507732.5505050505, "avg_output_tokens_per_task": 29977.13014763015, "last_run_timestamp": "2026-10-31T08:09:30.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.860269, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2318445, "avg_duration_per_task_ms": 364508.0101010101, "avg_output_tokens_per_task": 51949.767676767675, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.860269, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00987439393939394, "avg_duration_per_task_ms": 74831.28787878787, "avg_output_tokens_per_task": 9997.058080808081, "last_run_timestamp": "2026-10-24T08:14:36.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.859848375, "accuracy_std_dev": 0.022266733645381724, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13791987941919195, "avg_duration_per_task_ms": 1281636.821338384, "avg_output_tokens_per_task": 52297.90909090909, "last_run_timestamp": "2026-10-31T16:08:31.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8598483333333333, "accuracy_std_dev": 0.018068234858139545, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.8039200313552188, "avg_duration_per_task_ms": 487583.53682659933, "avg_output_tokens_per_task": 30465.617424242424, "last_run_timestamp": "2026-10-31T09:25:22.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8597737453957996, "accuracy_std_dev": 0.00946268128798041, "run_count": 19, "total_questions": 3095, "avg_cost_per_task_usd": 0.17348523166397414, "avg_duration_per_task_ms": 253011.81744749597, "avg_output_tokens_per_task": 31870.79353796446, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.859708, "accuracy_std_dev": 0.01589205338106649, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09237588832772166, "avg_duration_per_task_ms": 179383.0931537598, "avg_output_tokens_per_task": 25530.111672278337, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.8596746149425287, "accuracy_std_dev": 0.000831499999999985, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.0899417959770115, "avg_duration_per_task_ms": 1024745.4281609196, "avg_output_tokens_per_task": 35825.26724137931, "last_run_timestamp": "2026-10-27T13:57:40.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8594970187441424, "accuracy_std_dev": 0.03146274773954612, "run_count": 11, "total_questions": 2134, "avg_cost_per_task_usd": 0.1325802572633552, "avg_duration_per_task_ms": 467358.20337394567, "avg_output_tokens_per_task": 43781.12417994377, "last_run_timestamp": "2026-10-31T16:16:35.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.859429902017291, "accuracy_std_dev": 0.05098775585122179, "run_count": 11, "total_questions": 2082, "avg_cost_per_task_usd": 0.011825536023054757, "avg_duration_per_task_ms": 442610.81796349667, "avg_output_tokens_per_task": 25201.657060518733, "last_run_timestamp": "2026-10-31T08:27:46.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.859217125, "accuracy_std_dev": 0.010875888888241494, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.014054850378787878, "avg_duration_per_task_ms": 809652.5277777778, "avg_output_tokens_per_task": 28625.664772727272, "last_run_timestamp": "2026-10-31T16:17:15.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8590065833333335, "accuracy_std_dev": 0.008723943751331079, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8530048674242424, "avg_duration_per_task_ms": 481951.62794612796, "avg_output_tokens_per_task": 33929.37457912458, "last_run_timestamp": "2026-10-31T08:48:00.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8588278767123287, "accuracy_std_dev": 0.014585318196353787, "run_count": 3, "total_questions": 438, "avg_cost_per_task_usd": 0.005061143835616439, "avg_duration_per_task_ms": 414471.07305936073, "avg_output_tokens_per_task": 13714.753424657534, "last_run_timestamp": "2026-10-31T16:45:39.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.858165, "accuracy_std_dev": 0.026280391530568943, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07870610921717172, "avg_duration_per_task_ms": 466337.7960858586, "avg_output_tokens_per_task": 23353.214646464647, "last_run_timestamp": "2026-10-31T16:12:07.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8581648333333333, "accuracy_std_dev": 0.025909089369798823, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8336664709595959, "avg_duration_per_task_ms": 439138.7053872054, "avg_output_tokens_per_task": 33233.371632996634, "last_run_timestamp": "2026-10-31T08:35:35.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8580245833333333, "accuracy_std_dev": 0.017007195744048704, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8381309111952863, "avg_duration_per_task_ms": 457642.2281144781, "avg_output_tokens_per_task": 33349.60101010101, "last_run_timestamp": "2026-10-31T08:40:44.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8579124, "accuracy_std_dev": 0.017384854495796048, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02057177575757576, "avg_duration_per_task_ms": 445221.4888888889, "avg_output_tokens_per_task": 40903.725252525255, "last_run_timestamp": "2026-10-31T08:25:16.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8572705625000001, "accuracy_std_dev": 0.007924119438214786, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.06404825631313131, "avg_duration_per_task_ms": 595272.8756313132, "avg_output_tokens_per_task": 35321.874684343435, "last_run_timestamp": "2026-10-31T16:08:56.835Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8570892222222223, "accuracy_std_dev": 0.031137791974732713, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.015986119528619527, "avg_duration_per_task_ms": 333449.29124579125, "avg_output_tokens_per_task": 28587.83613916947, "last_run_timestamp": "2026-10-31T16:04:48.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.8569025, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018898722222222223, "avg_duration_per_task_ms": 164757.63383838383, "avg_output_tokens_per_task": 31337.035353535353, "last_run_timestamp": "2026-10-24T08:17:56.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8568698741301679, "accuracy_std_dev": 0.08960692857100729, "run_count": 25, "total_questions": 4886, "avg_cost_per_task_usd": 0.00782735918952108, "avg_duration_per_task_ms": 558967.4410560786, "avg_output_tokens_per_task": 27058.752762996315, "last_run_timestamp": "2026-11-06T22:28:22.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8568247201068804, "accuracy_std_dev": 0.015059304005690374, "run_count": 9, "total_questions": 1497, "avg_cost_per_task_usd": 0.004656902471609886, "avg_duration_per_task_ms": 413401.8376753507, "avg_output_tokens_per_task": 9906.163660654642, "last_run_timestamp": "2026-10-31T09:01:06.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8566210871819038, "accuracy_std_dev": 0.02083134388139214, "run_count": 43, "total_questions": 8488, "avg_cost_per_task_usd": 0.2924742340951932, "avg_duration_per_task_ms": 342907.9387370405, "avg_output_tokens_per_task": 52593.62170122526, "last_run_timestamp": "2026-10-31T16:26:37.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8564403552465234, "accuracy_std_dev": 0.02676663000516378, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.08082342730720607, "avg_duration_per_task_ms": 485283.54403708386, "avg_output_tokens_per_task": 41227.2022756005, "last_run_timestamp": "2026-10-31T16:30:10.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8563411666666667, "accuracy_std_dev": 0.009991279087562064, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005837240740740741, "avg_duration_per_task_ms": 544489.4410774411, "avg_output_tokens_per_task": 31556.968855218856, "last_run_timestamp": "2026-10-31T16:16:55.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.855850125, "accuracy_std_dev": 0.029146120652830884, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09797882512626263, "avg_duration_per_task_ms": 344142.03661616164, "avg_output_tokens_per_task": 21821.31313131313, "last_run_timestamp": "2026-10-31T16:15:29.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8558501249999999, "accuracy_std_dev": 0.0195049467087038, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04610065845959596, "avg_duration_per_task_ms": 601194.1792929292, "avg_output_tokens_per_task": 37602.47222222222, "last_run_timestamp": "2026-10-31T16:03:21.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8556823130287649, "accuracy_std_dev": 0.021169598748854452, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.08525841285956007, "avg_duration_per_task_ms": 295845.3159898477, "avg_output_tokens_per_task": 25006.826565143823, "last_run_timestamp": "2026-10-31T08:34:08.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8556049172108208, "accuracy_std_dev": 0.03188071644780284, "run_count": 22, "total_questions": 4288, "avg_cost_per_task_usd": 0.003744977378731343, "avg_duration_per_task_ms": 73667.2943097015, "avg_output_tokens_per_task": 13976.44776119403, "last_run_timestamp": "2026-10-31T08:12:47.827Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8554592857142856, "accuracy_std_dev": 0.010776903155946642, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.14148054425204423, "avg_duration_per_task_ms": 433829.56758056756, "avg_output_tokens_per_task": 44977.72101972102, "last_run_timestamp": "2026-10-31T16:26:59.503Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8552186666666667, "accuracy_std_dev": 0.0148680429631998, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007792587542087542, "avg_duration_per_task_ms": 536143.4478114478, "avg_output_tokens_per_task": 28061.833333333332, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8550504000000001, "accuracy_std_dev": 0.012697892748011365, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.029575816161616166, "avg_duration_per_task_ms": 786613.2691919192, "avg_output_tokens_per_task": 33730.74898989899, "last_run_timestamp": "2026-10-17T08:19:33.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.8549781428571429, "accuracy_std_dev": 0.06322451474914856, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.16852446320346318, "avg_duration_per_task_ms": 1962682.6176046175, "avg_output_tokens_per_task": 59425.0505050505, "last_run_timestamp": "2026-10-24T16:14:24.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8546926249999999, "accuracy_std_dev": 0.022345683453060337, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04371308901515151, "avg_duration_per_task_ms": 649064.4501262626, "avg_output_tokens_per_task": 36365.37247474748, "last_run_timestamp": "2026-10-31T16:02:14.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8545531474226805, "accuracy_std_dev": 0.018117679996070137, "run_count": 10, "total_questions": 1940, "avg_cost_per_task_usd": 0.08358154742268041, "avg_duration_per_task_ms": 325785.14793814434, "avg_output_tokens_per_task": 28959.811340206186, "last_run_timestamp": "2026-10-31T08:56:36.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8540963333333333, "accuracy_std_dev": 0.019358062075872505, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03021066049382716, "avg_duration_per_task_ms": 606023.7901234567, "avg_output_tokens_per_task": 34216.56677890011, "last_run_timestamp": "2026-10-31T16:02:14.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8540087088607595, "accuracy_std_dev": 0.011340000000000017, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.023069045569620254, "avg_duration_per_task_ms": 780937.6962025317, "avg_output_tokens_per_task": 22650.07341772152, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8537037, "accuracy_std_dev": 0.012630753865466617, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10325877878787879, "avg_duration_per_task_ms": 455294.6515151515, "avg_output_tokens_per_task": 31795.09494949495, "last_run_timestamp": "2026-10-31T08:16:11.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8536755833333335, "accuracy_std_dev": 0.03297161670148616, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7094076557239056, "avg_duration_per_task_ms": 485859.81018518517, "avg_output_tokens_per_task": 28361.97053872054, "last_run_timestamp": "2026-10-31T08:09:42.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8535090068846816, "accuracy_std_dev": 0.005804304535067259, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.09211812679288582, "avg_duration_per_task_ms": 680955.4825014343, "avg_output_tokens_per_task": 42013.00631095812, "last_run_timestamp": "2026-10-31T16:11:17.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8530642646239555, "accuracy_std_dev": 0.022329477337983097, "run_count": 4, "total_questions": 718, "avg_cost_per_task_usd": 0.02860370194986072, "avg_duration_per_task_ms": 460751.2409470752, "avg_output_tokens_per_task": 34791.85933147632, "last_run_timestamp": "2026-10-10T08:22:41.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.852836585106383, "accuracy_std_dev": 0.00992599999999999, "run_count": 2, "total_questions": 376, "avg_cost_per_task_usd": 0.10881258244680853, "avg_duration_per_task_ms": 389377.2340425532, "avg_output_tokens_per_task": 27704.662234042553, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.8523164263322883, "accuracy_std_dev": 0.010034068893524688, "run_count": 5, "total_questions": 957, "avg_cost_per_task_usd": 0.08246928213166145, "avg_duration_per_task_ms": 272317.5339602926, "avg_output_tokens_per_task": 27908.60710553814, "last_run_timestamp": "2026-10-31T08:44:39.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8521639072512648, "accuracy_std_dev": 0.012873132633608224, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.08519417144463183, "avg_duration_per_task_ms": 288650.1905564924, "avg_output_tokens_per_task": 29819.736368746486, "last_run_timestamp": "2026-10-31T08:22:52.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8521324166666666, "accuracy_std_dev": 0.01894507738375299, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12555288047138044, "avg_duration_per_task_ms": 374701.57112794614, "avg_output_tokens_per_task": 39360.16035353536, "last_run_timestamp": "2026-10-31T16:08:08.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.8521323333333334, "accuracy_std_dev": 0.03225728546993512, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005255346801346801, "avg_duration_per_task_ms": 1011045.1414141414, "avg_output_tokens_per_task": 28586.09259259259, "last_run_timestamp": "2026-10-31T16:46:54.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8521322499999999, "accuracy_std_dev": 0.047036847074261895, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.16922138846801346, "avg_duration_per_task_ms": 167377.74537037036, "avg_output_tokens_per_task": 16702.02441077441, "last_run_timestamp": "2026-10-31T08:12:45.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.8521062280219781, "accuracy_std_dev": 0.006055500000000047, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.005501717032967033, "avg_duration_per_task_ms": 539158.467032967, "avg_output_tokens_per_task": 15491.851648351649, "last_run_timestamp": "2026-10-17T08:17:58.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8520922857142857, "accuracy_std_dev": 0.010237499481448224, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014528669552669555, "avg_duration_per_task_ms": 664432.2056277057, "avg_output_tokens_per_task": 19049.7316017316, "last_run_timestamp": "2026-10-31T16:05:36.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.851852, "accuracy_std_dev": 0.005323694440893428, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.011784135101010102, "avg_duration_per_task_ms": 222866.32323232322, "avg_output_tokens_per_task": 29178.76893939394, "last_run_timestamp": "2026-11-04T18:07:25.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8514385818947369, "accuracy_std_dev": 0.00836367476510108, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.03818178821052631, "avg_duration_per_task_ms": 384700.4193684211, "avg_output_tokens_per_task": 38833.43747368421, "last_run_timestamp": "2026-10-31T16:26:59.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8513607500000001, "accuracy_std_dev": 0.010092092937253725, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09869350589225588, "avg_duration_per_task_ms": 554753.9515993266, "avg_output_tokens_per_task": 30538.262205387204, "last_run_timestamp": "2026-10-31T16:01:53.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.8512906666666668, "accuracy_std_dev": 0.00555530025191155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017986696969696972, "avg_duration_per_task_ms": 129036.91750841751, "avg_output_tokens_per_task": 29778.104377104377, "last_run_timestamp": "2026-10-31T08:56:30.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8512828567483604, "accuracy_std_dev": 0.02197685353720036, "run_count": 15, "total_questions": 2897, "avg_cost_per_task_usd": 0.21227700207110806, "avg_duration_per_task_ms": 253125.91128753882, "avg_output_tokens_per_task": 43881.1304798067, "last_run_timestamp": "2026-10-24T16:29:51.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.850783975720789, "accuracy_std_dev": 0.012623895335434301, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.03144315275670207, "avg_duration_per_task_ms": 755960.0136570561, "avg_output_tokens_per_task": 32811.26707132018, "last_run_timestamp": "2026-10-17T08:19:39.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8502175226400613, "accuracy_std_dev": 0.012397473946332766, "run_count": 8, "total_questions": 1303, "avg_cost_per_task_usd": 0.00643011281657713, "avg_duration_per_task_ms": 366999.48887183424, "avg_output_tokens_per_task": 15845.35226400614, "last_run_timestamp": "2026-10-31T16:08:33.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8502042599087539, "accuracy_std_dev": 0.01590746948888509, "run_count": 37, "total_questions": 7014, "avg_cost_per_task_usd": 0.16371538508696884, "avg_duration_per_task_ms": 188635.26946107784, "avg_output_tokens_per_task": 29341.152266894784, "last_run_timestamp": "2026-10-31T09:24:22.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8501683333333333, "accuracy_std_dev": 0.004956089811759068, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05497935774410773, "avg_duration_per_task_ms": 419820.80387205386, "avg_output_tokens_per_task": 28107.88973063973, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.850168, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.09743340740740741, "avg_duration_per_task_ms": 424537.84175084176, "avg_output_tokens_per_task": 31131.93771043771, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.84974725, "accuracy_std_dev": 0.054530591026390865, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005316606060606061, "avg_duration_per_task_ms": 207511.0707070707, "avg_output_tokens_per_task": 18629.959595959597, "last_run_timestamp": "2026-11-05T15:00:03.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "NextBit", "accuracy": 0.8495076506329113, "accuracy_std_dev": 0.005079115528864803, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.007176422784810127, "avg_duration_per_task_ms": 1312452.7088607594, "avg_output_tokens_per_task": 17778.172151898732, "last_run_timestamp": "2026-11-02T17:22:59.212Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8494951090909092, "accuracy_std_dev": 0.012579000000000007, "run_count": 2, "total_questions": 165, "avg_cost_per_task_usd": 0.005448739393939394, "avg_duration_per_task_ms": 236133.1696969697, "avg_output_tokens_per_task": 25659.842424242423, "last_run_timestamp": "2026-10-31T16:10:36.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8492564166666666, "accuracy_std_dev": 0.015991238316957257, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1367386346801347, "avg_duration_per_task_ms": 450843.9692760943, "avg_output_tokens_per_task": 49203.65193602694, "last_run_timestamp": "2026-10-31T16:09:19.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8492264405594405, "accuracy_std_dev": 0.009705494209950108, "run_count": 8, "total_questions": 1573, "avg_cost_per_task_usd": 0.007261696757787667, "avg_duration_per_task_ms": 592695.9872854418, "avg_output_tokens_per_task": 17964.459631277812, "last_run_timestamp": "2026-10-31T16:12:28.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "StreamLake", "accuracy": 0.8491083950617284, "accuracy_std_dev": 0.03011880589939707, "run_count": 6, "total_questions": 486, "avg_cost_per_task_usd": 0.026545792181069956, "avg_duration_per_task_ms": 90656.26131687243, "avg_output_tokens_per_task": 8757.810699588477, "last_run_timestamp": "2026-11-02T17:24:29.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8490562452830188, "accuracy_std_dev": 0.004459999999999964, "run_count": 2, "total_questions": 265, "avg_cost_per_task_usd": 0.017200528301886793, "avg_duration_per_task_ms": 165535.98490566039, "avg_output_tokens_per_task": 28515.603773584906, "last_run_timestamp": "2026-10-31T08:20:07.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.84890575, "accuracy_std_dev": 0.05035112340045156, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1629483771043771, "avg_duration_per_task_ms": 162891.44065656565, "avg_output_tokens_per_task": 16117.493265993266, "last_run_timestamp": "2026-10-31T08:45:37.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8489055833333334, "accuracy_std_dev": 0.01662490835793455, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.052647822390572384, "avg_duration_per_task_ms": 274255.4238215488, "avg_output_tokens_per_task": 45768.43518518518, "last_run_timestamp": "2026-10-31T16:53:23.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.8488202684365781, "accuracy_std_dev": 0.035164132211494975, "run_count": 4, "total_questions": 678, "avg_cost_per_task_usd": 0.027723622418879056, "avg_duration_per_task_ms": 605422.2300884955, "avg_output_tokens_per_task": 13768.485250737464, "last_run_timestamp": "2026-10-17T08:18:07.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.019539080808080807, "avg_duration_per_task_ms": 826109.8787878788, "avg_output_tokens_per_task": 23180.459595959597, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.84811653125, "accuracy_std_dev": 0.0186791945439712, "run_count": 32, "total_questions": 6336, "avg_cost_per_task_usd": 0.2417097290088384, "avg_duration_per_task_ms": 494923.91840277775, "avg_output_tokens_per_task": 47859.75946969697, "last_run_timestamp": "2026-10-31T17:01:17.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8480038571428572, "accuracy_std_dev": 0.046172433270539756, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09876766017316017, "avg_duration_per_task_ms": 1234895.2164502165, "avg_output_tokens_per_task": 39606.40476190476, "last_run_timestamp": "2026-10-31T16:01:53.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.847949384816754, "accuracy_std_dev": 0.009393024418270302, "run_count": 12, "total_questions": 2292, "avg_cost_per_task_usd": 0.06595389223385689, "avg_duration_per_task_ms": 190454.0772251309, "avg_output_tokens_per_task": 20514.38787085515, "last_run_timestamp": "2026-10-31T16:08:31.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.8479167041666666, "accuracy_std_dev": 0.00828816902819917, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.16138468541666665, "avg_duration_per_task_ms": 277988.090625, "avg_output_tokens_per_task": 24274.81875, "last_run_timestamp": "2026-10-28T05:08:57.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8477888928571429, "accuracy_std_dev": 0.029215151613674717, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.1592395025510204, "avg_duration_per_task_ms": 438119.2229591837, "avg_output_tokens_per_task": 36131.57244897959, "last_run_timestamp": "2026-10-31T08:13:34.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8473217418810629, "accuracy_std_dev": 0.031381320315484754, "run_count": 12, "total_questions": 2371, "avg_cost_per_task_usd": 0.009025258118937158, "avg_duration_per_task_ms": 479873.75579924084, "avg_output_tokens_per_task": 32127.526360185577, "last_run_timestamp": "2026-10-31T08:31:37.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8469361045003815, "accuracy_std_dev": 0.04231320024956013, "run_count": 8, "total_questions": 1311, "avg_cost_per_task_usd": 0.02445259496567506, "avg_duration_per_task_ms": 437957.52021357743, "avg_output_tokens_per_task": 38826.28070175439, "last_run_timestamp": "2026-10-31T08:55:46.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.8468582586644126, "accuracy_std_dev": 0.013598361278927057, "run_count": 6, "total_questions": 1183, "avg_cost_per_task_usd": 0.26757311665257816, "avg_duration_per_task_ms": 498008.1420118343, "avg_output_tokens_per_task": 60614.17836010144, "last_run_timestamp": "2026-10-31T16:45:45.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.84680125, "accuracy_std_dev": 0.03303257248440542, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.011049029040404038, "avg_duration_per_task_ms": 454169.0126262626, "avg_output_tokens_per_task": 39139.167297979795, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8466142222222222, "accuracy_std_dev": 0.010836379374616662, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14184849326599328, "avg_duration_per_task_ms": 144926.00224466893, "avg_output_tokens_per_task": 32057.991582491584, "last_run_timestamp": "2026-10-31T08:40:49.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8465387902641119, "accuracy_std_dev": 0.025032600265254102, "run_count": 10, "total_questions": 1931, "avg_cost_per_task_usd": 0.10684122164681513, "avg_duration_per_task_ms": 489966.58933195233, "avg_output_tokens_per_task": 34902.65044018643, "last_run_timestamp": "2026-10-31T09:15:06.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8459595000000001, "accuracy_std_dev": 0.010636232717304867, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.019541941077441075, "avg_duration_per_task_ms": 139818.19360269362, "avg_output_tokens_per_task": 19064.3265993266, "last_run_timestamp": "2026-10-31T16:05:14.635Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8459293152073732, "accuracy_std_dev": 0.017246738162321286, "run_count": 11, "total_questions": 2170, "avg_cost_per_task_usd": 0.010452895391705068, "avg_duration_per_task_ms": 454110.96359447006, "avg_output_tokens_per_task": 34047.99170506912, "last_run_timestamp": "2026-10-31T08:27:51.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8456636089524402, "accuracy_std_dev": 0.018418318454142355, "run_count": 17, "total_questions": 3217, "avg_cost_per_task_usd": 0.14255490332608023, "avg_duration_per_task_ms": 266408.3546782717, "avg_output_tokens_per_task": 28803.044140503574, "last_run_timestamp": "2026-10-31T08:33:10.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8446468343465044, "accuracy_std_dev": 0.01783018725336332, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.0660811013171226, "avg_duration_per_task_ms": 245353.75430597772, "avg_output_tokens_per_task": 20336.286220871327, "last_run_timestamp": "2026-10-31T08:32:53.731Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8446020893155258, "accuracy_std_dev": 0.020816297010995962, "run_count": 8, "total_questions": 1198, "avg_cost_per_task_usd": 0.018853567612687815, "avg_duration_per_task_ms": 104380.7020033389, "avg_output_tokens_per_task": 16184.987479131887, "last_run_timestamp": "2026-10-31T16:06:00.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Modal", "accuracy": 0.844322, "accuracy_std_dev": null, "run_count": 1, "total_questions": 182, "avg_cost_per_task_usd": 0.0016337307692307693, "avg_duration_per_task_ms": 52884.66483516483, "avg_output_tokens_per_task": 15097.615384615385, "last_run_timestamp": "2026-11-04T09:00:10.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.844276, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036391522727272724, "avg_duration_per_task_ms": 53072.40404040404, "avg_output_tokens_per_task": 14468.015151515152, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8439077604166666, "accuracy_std_dev": 0.009070260174817954, "run_count": 12, "total_questions": 2112, "avg_cost_per_task_usd": 0.12450989630681818, "avg_duration_per_task_ms": 153233.97348484848, "avg_output_tokens_per_task": 31747.522727272728, "last_run_timestamp": "2026-10-31T08:45:40.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8438831999999998, "accuracy_std_dev": 0.010796604899689534, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.1486385685185185, "avg_duration_per_task_ms": 173235.29764309764, "avg_output_tokens_per_task": 32642.098316498315, "last_run_timestamp": "2026-10-31T08:24:42.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8434343333333334, "accuracy_std_dev": 0.0404743207505972, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.12289456902356903, "avg_duration_per_task_ms": 641373.1111111111, "avg_output_tokens_per_task": 41514.85353535353, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8434174840336134, "accuracy_std_dev": 0.0058964576706039785, "run_count": 7, "total_questions": 1190, "avg_cost_per_task_usd": 0.07875789159663865, "avg_duration_per_task_ms": 236163.5705882353, "avg_output_tokens_per_task": 19324.579831932773, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8433784397283531, "accuracy_std_dev": 0.003767029717016947, "run_count": 6, "total_questions": 1178, "avg_cost_per_task_usd": 0.1209133650254669, "avg_duration_per_task_ms": 239041.5050933786, "avg_output_tokens_per_task": 32072.49235993209, "last_run_timestamp": "2026-10-31T09:12:37.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8433156148597423, "accuracy_std_dev": 0.007093117054653797, "run_count": 7, "total_questions": 1319, "avg_cost_per_task_usd": 0.05370390523123579, "avg_duration_per_task_ms": 267310.77255496586, "avg_output_tokens_per_task": 22285.294920394237, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.843159054532056, "accuracy_std_dev": 0.012250910211082282, "run_count": 8, "total_questions": 1357, "avg_cost_per_task_usd": 0.0069029462048636695, "avg_duration_per_task_ms": 626843.2291820191, "avg_output_tokens_per_task": 17018.759027266027, "last_run_timestamp": "2026-10-31T16:34:27.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8430962619246861, "accuracy_std_dev": 0.029385860831636818, "run_count": 7, "total_questions": 1195, "avg_cost_per_task_usd": 0.08830847447698745, "avg_duration_per_task_ms": 1349078.5882845188, "avg_output_tokens_per_task": 40536.07280334728, "last_run_timestamp": "2026-10-31T16:04:23.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8430133333333335, "accuracy_std_dev": 0.021951449114402964, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05437653409090909, "avg_duration_per_task_ms": 247748.2095959596, "avg_output_tokens_per_task": 23043.897727272728, "last_run_timestamp": "2026-10-31T16:08:08.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8426947486187845, "accuracy_std_dev": 0.027181441039645704, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.01345692311233886, "avg_duration_per_task_ms": 682984.7329650092, "avg_output_tokens_per_task": 42267.71869244936, "last_run_timestamp": "2026-10-31T08:57:00.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8425924999999999, "accuracy_std_dev": 0.015993499999999994, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006982, "avg_duration_per_task_ms": 216832.21464646465, "avg_output_tokens_per_task": 14749.073232323231, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.8425923750000001, "accuracy_std_dev": 0.07901703473450755, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.06442433396464646, "avg_duration_per_task_ms": 1182218.1875, "avg_output_tokens_per_task": 33113.868055555555, "last_run_timestamp": "2026-10-31T16:05:14.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8423118888888889, "accuracy_std_dev": 0.04281090341242117, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0038540690235690234, "avg_duration_per_task_ms": 561172.7755331089, "avg_output_tokens_per_task": 41338.6531986532, "last_run_timestamp": "2026-10-31T16:18:02.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8422337612565445, "accuracy_std_dev": 0.023212455418589382, "run_count": 10, "total_questions": 1910, "avg_cost_per_task_usd": 0.10300291204188483, "avg_duration_per_task_ms": 203070.60157068062, "avg_output_tokens_per_task": 30819.346073298428, "last_run_timestamp": "2026-10-31T08:52:06.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.842080958563536, "accuracy_std_dev": 0.01924201603413352, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.02133896546961326, "avg_duration_per_task_ms": 337269.2900552486, "avg_output_tokens_per_task": 42459.25598526704, "last_run_timestamp": "2026-10-31T08:44:51.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.841751, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.15673991919191918, "avg_duration_per_task_ms": 1499085.4595959596, "avg_output_tokens_per_task": 52100.893939393936, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.841751, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06159810101010101, "avg_duration_per_task_ms": 79946.62794612795, "avg_output_tokens_per_task": 13559.319865319865, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8416756038338659, "accuracy_std_dev": 0.009989855274226949, "run_count": 5, "total_questions": 939, "avg_cost_per_task_usd": 0.01604648136315229, "avg_duration_per_task_ms": 662324.1448349308, "avg_output_tokens_per_task": 17011.09478168264, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8416364037363778, "accuracy_std_dev": 0.015589563439410502, "run_count": 11, "total_questions": 1927, "avg_cost_per_task_usd": 0.1375403404255319, "avg_duration_per_task_ms": 272709.34509600414, "avg_output_tokens_per_task": 32850.36170212766, "last_run_timestamp": "2026-10-31T09:42:32.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8414702500000001, "accuracy_std_dev": 0.0327729719792723, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05281460311447811, "avg_duration_per_task_ms": 205567.1611952862, "avg_output_tokens_per_task": 21904.65446127946, "last_run_timestamp": "2026-10-31T16:28:22.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8413764444444444, "accuracy_std_dev": 0.014280020589085009, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13749752525252526, "avg_duration_per_task_ms": 888166.4298540965, "avg_output_tokens_per_task": 54880.480359147026, "last_run_timestamp": "2026-10-31T16:10:36.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "PrimeIntellect", "accuracy": 0.841228, "accuracy_std_dev": null, "run_count": 1, "total_questions": 190, "avg_cost_per_task_usd": 0.0038195157894736842, "avg_duration_per_task_ms": 400775.6, "avg_output_tokens_per_task": 17024.936842105264, "last_run_timestamp": "2026-11-01T18:00:07.350Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8411896666666668, "accuracy_std_dev": 0.025531658844135698, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.16060966610549945, "avg_duration_per_task_ms": 1022232.3557800225, "avg_output_tokens_per_task": 52797.9152637486, "last_run_timestamp": "2026-10-31T16:39:29.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8409089999999999, "accuracy_std_dev": 0.010943000000000036, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03708839646464646, "avg_duration_per_task_ms": 47515.843434343435, "avg_output_tokens_per_task": 14746.045454545454, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.840891889212828, "accuracy_std_dev": 0.01539818649719325, "run_count": 16, "total_questions": 3087, "avg_cost_per_task_usd": 0.022577127632005183, "avg_duration_per_task_ms": 250189.5425979916, "avg_output_tokens_per_task": 38449.29964366699, "last_run_timestamp": "2026-10-31T08:54:15.638Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8408359820854386, "accuracy_std_dev": 0.011558364030339736, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.14478948920532844, "avg_duration_per_task_ms": 203069.29581993568, "avg_output_tokens_per_task": 32673.29122645843, "last_run_timestamp": "2026-10-31T09:21:48.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8407407, "accuracy_std_dev": 0.024165205569371837, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.013536882323232325, "avg_duration_per_task_ms": 528967.1176767676, "avg_output_tokens_per_task": 40146.823737373736, "last_run_timestamp": "2026-10-31T08:45:25.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8406767200863932, "accuracy_std_dev": 0.03362626844282443, "run_count": 12, "total_questions": 2315, "avg_cost_per_task_usd": 0.030011868250539957, "avg_duration_per_task_ms": 286135.6241900648, "avg_output_tokens_per_task": 9270.458315334774, "last_run_timestamp": "2026-10-31T08:16:12.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8406685714285715, "accuracy_std_dev": 0.01850043854968342, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02123720418470418, "avg_duration_per_task_ms": 436896.4862914863, "avg_output_tokens_per_task": 42244.7886002886, "last_run_timestamp": "2026-10-31T08:09:30.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.8404879999999999, "accuracy_std_dev": 0.020003728864889173, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037026556818181816, "avg_duration_per_task_ms": 113840.68308080808, "avg_output_tokens_per_task": 34919.68308080808, "last_run_timestamp": "2026-10-31T09:18:03.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8403390039479797, "accuracy_std_dev": 0.012076015641029829, "run_count": 22, "total_questions": 4306, "avg_cost_per_task_usd": 0.10892927728750582, "avg_duration_per_task_ms": 343834.3037621923, "avg_output_tokens_per_task": 34310.84417092429, "last_run_timestamp": "2026-11-06T23:05:09.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.8401781603650587, "accuracy_std_dev": 0.014388231606073072, "run_count": 16, "total_questions": 3068, "avg_cost_per_task_usd": 0.07506749576271186, "avg_duration_per_task_ms": 388300.5521512386, "avg_output_tokens_per_task": 22304.916558018253, "last_run_timestamp": "2026-10-31T08:43:48.577Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.8398567499999999, "accuracy_std_dev": 0.010281237665646122, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005851238636363636, "avg_duration_per_task_ms": 457462.80303030304, "avg_output_tokens_per_task": 25951.3125, "last_run_timestamp": "2026-10-31T16:45:40.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8398265714285714, "accuracy_std_dev": 0.008557108588054464, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03983322727272727, "avg_duration_per_task_ms": 52955.70346320346, "avg_output_tokens_per_task": 14785.20634920635, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.839812126197183, "accuracy_std_dev": 0.07223149496882991, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.06668295830985914, "avg_duration_per_task_ms": 1704327.0591549296, "avg_output_tokens_per_task": 20074.104225352112, "last_run_timestamp": "2026-10-31T16:13:09.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8393658333333334, "accuracy_std_dev": 0.01474792674600588, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.23697320664983168, "avg_duration_per_task_ms": 282494.2058080808, "avg_output_tokens_per_task": 53697.90698653198, "last_run_timestamp": "2026-10-31T16:57:18.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8391730814863104, "accuracy_std_dev": 0.011927781960856079, "run_count": 31, "total_questions": 6136, "avg_cost_per_task_usd": 0.26252152737940027, "avg_duration_per_task_ms": 275082.67649934813, "avg_output_tokens_per_task": 57033.21740547588, "last_run_timestamp": "2026-10-31T16:16:55.693Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8387440275454993, "accuracy_std_dev": 0.019690853286246784, "run_count": 11, "total_questions": 2033, "avg_cost_per_task_usd": 0.07400631874077719, "avg_duration_per_task_ms": 403244.14658140676, "avg_output_tokens_per_task": 24425.756025577964, "last_run_timestamp": "2026-10-31T16:18:04.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8387141903143586, "accuracy_std_dev": 0.029140860908219966, "run_count": 12, "total_questions": 2354, "avg_cost_per_task_usd": 0.0665123759558199, "avg_duration_per_task_ms": 1085367.209855565, "avg_output_tokens_per_task": 33157.14528462192, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8386898181818183, "accuracy_std_dev": 0.022320469186247312, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1030642401285583, "avg_duration_per_task_ms": 384608.90220385673, "avg_output_tokens_per_task": 35835.99357208448, "last_run_timestamp": "2026-10-31T08:36:48.545Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8383837142857142, "accuracy_std_dev": 0.012139811903865117, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1527254054834055, "avg_duration_per_task_ms": 220532.05050505052, "avg_output_tokens_per_task": 34470.08658008658, "last_run_timestamp": "2026-10-31T08:45:52.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8383836666666667, "accuracy_std_dev": 0.006299258386685094, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02860740067340067, "avg_duration_per_task_ms": 515900.1077441077, "avg_output_tokens_per_task": 23652.510101010103, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.838173375, "accuracy_std_dev": 0.011212592975506409, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09136887752525254, "avg_duration_per_task_ms": 1122835.0227272727, "avg_output_tokens_per_task": 46187.42108585859, "last_run_timestamp": "2026-10-31T16:33:40.856Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8378226666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.15828378956228956, "avg_duration_per_task_ms": 446608.7861952862, "avg_output_tokens_per_task": 56298.93097643097, "last_run_timestamp": "2026-10-10T08:30:28.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8376822500000001, "accuracy_std_dev": 0.06783075684024788, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.20651400042087545, "avg_duration_per_task_ms": 551769.9718013467, "avg_output_tokens_per_task": 46912.232323232325, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.837542, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06186689898989899, "avg_duration_per_task_ms": 75089.61363636363, "avg_output_tokens_per_task": 13620.507575757576, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8375419999999999, "accuracy_std_dev": 0.012551288592544344, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.08632508417508418, "avg_duration_per_task_ms": 711440.0951178451, "avg_output_tokens_per_task": 49539.13468013468, "last_run_timestamp": "2026-10-31T16:46:10.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8371593636363637, "accuracy_std_dev": 0.013027195241792026, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.141865238292011, "avg_duration_per_task_ms": 181084.47015610652, "avg_output_tokens_per_task": 32015.05968778696, "last_run_timestamp": "2026-10-31T08:16:11.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.8371268528072838, "accuracy_std_dev": 0.033679846992081194, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.1096482544258978, "avg_duration_per_task_ms": 1061118.531613556, "avg_output_tokens_per_task": 34187.799696509865, "last_run_timestamp": "2026-10-24T16:17:57.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.8369882061403507, "accuracy_std_dev": 0.09929590339099784, "run_count": 8, "total_questions": 1368, "avg_cost_per_task_usd": 0.042823769005847945, "avg_duration_per_task_ms": 127078.64692982456, "avg_output_tokens_per_task": 28056.13523391813, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8367623703349283, "accuracy_std_dev": 0.019945105490341078, "run_count": 12, "total_questions": 2090, "avg_cost_per_task_usd": 0.09667845789473684, "avg_duration_per_task_ms": 610571.3789473685, "avg_output_tokens_per_task": 30136.36889952153, "last_run_timestamp": "2026-10-31T16:13:36.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.83672117730096, "accuracy_std_dev": 0.04633394919974385, "run_count": 9, "total_questions": 1771, "avg_cost_per_task_usd": 0.192393172783738, "avg_duration_per_task_ms": 603566.0841332581, "avg_output_tokens_per_task": 45791.96103896104, "last_run_timestamp": "2026-10-31T16:25:21.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8367003333333334, "accuracy_std_dev": 0.04194124922899757, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.09553346127946127, "avg_duration_per_task_ms": 1453001.978114478, "avg_output_tokens_per_task": 42378.70538720539, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8365599166666667, "accuracy_std_dev": 0.02438133608746088, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1378553611111111, "avg_duration_per_task_ms": 542424.4175084175, "avg_output_tokens_per_task": 43872.40151515151, "last_run_timestamp": "2026-10-31T16:18:28.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8361277244542866, "accuracy_std_dev": 0.026614847390527968, "run_count": 16, "total_questions": 3161, "avg_cost_per_task_usd": 0.02298093546346093, "avg_duration_per_task_ms": 307170.6646630813, "avg_output_tokens_per_task": 18491.486871243276, "last_run_timestamp": "2026-10-31T08:31:11.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8356791761071061, "accuracy_std_dev": 0.01579187012442372, "run_count": 15, "total_questions": 2913, "avg_cost_per_task_usd": 0.29531686096807414, "avg_duration_per_task_ms": 404294.6398901476, "avg_output_tokens_per_task": 50028.10332990045, "last_run_timestamp": "2026-10-31T16:56:08.871Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8348581735005084, "accuracy_std_dev": 0.011926219294758369, "run_count": 18, "total_questions": 2951, "avg_cost_per_task_usd": 0.1190305462555066, "avg_duration_per_task_ms": 291312.74483226024, "avg_output_tokens_per_task": 28021.837004405286, "last_run_timestamp": "2026-10-31T08:09:30.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8347361666666666, "accuracy_std_dev": 0.01234876949897801, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04779885185185185, "avg_duration_per_task_ms": 978213.5824915825, "avg_output_tokens_per_task": 29118.793771043773, "last_run_timestamp": "2026-10-17T08:20:47.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8346745432595574, "accuracy_std_dev": 0.02838986518520959, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.006750653923541246, "avg_duration_per_task_ms": 767094.1871227365, "avg_output_tokens_per_task": 18119.238095238095, "last_run_timestamp": "2026-10-31T16:04:29.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.8346120933503837, "accuracy_std_dev": 0.029860364128538167, "run_count": 6, "total_questions": 782, "avg_cost_per_task_usd": 0.09530405242966751, "avg_duration_per_task_ms": 330065.4283887468, "avg_output_tokens_per_task": 39472.66240409207, "last_run_timestamp": "2026-10-31T16:08:31.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.8344576188870152, "accuracy_std_dev": 0.007843600547934322, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.033416617200674535, "avg_duration_per_task_ms": 208747.37942664418, "avg_output_tokens_per_task": 14517.00168634064, "last_run_timestamp": "2026-10-31T08:47:23.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8344556666666667, "accuracy_std_dev": 0.02104175490674567, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.21689809427609427, "avg_duration_per_task_ms": 815662.5723905724, "avg_output_tokens_per_task": 49349.5, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8341543388067869, "accuracy_std_dev": 0.008301927538228698, "run_count": 10, "total_questions": 1827, "avg_cost_per_task_usd": 0.0431522140120416, "avg_duration_per_task_ms": 94743.94636015326, "avg_output_tokens_per_task": 15061.821018062397, "last_run_timestamp": "2026-10-31T08:22:53.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8341254705882353, "accuracy_std_dev": 0.010649185432078013, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.25025737195484254, "avg_duration_per_task_ms": 384709.65567439096, "avg_output_tokens_per_task": 40253.002970885325, "last_run_timestamp": "2026-10-31T08:41:02.403Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8339644999999999, "accuracy_std_dev": 0.02083967769064579, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09829379103535353, "avg_duration_per_task_ms": 1158045.8926767677, "avg_output_tokens_per_task": 45010.31502525252, "last_run_timestamp": "2026-10-31T16:08:56.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.833754, "accuracy_std_dev": 0.016681719388300487, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.14335061994949497, "avg_duration_per_task_ms": 311316.28472222225, "avg_output_tokens_per_task": 59097.76136363636, "last_run_timestamp": "2026-10-31T16:45:48.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8335494788062284, "accuracy_std_dev": 0.010937881698736537, "run_count": 12, "total_questions": 2312, "avg_cost_per_task_usd": 0.23824533304498272, "avg_duration_per_task_ms": 303110.6955017301, "avg_output_tokens_per_task": 53975.97967128028, "last_run_timestamp": "2026-10-31T16:28:00.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8328153846153846, "accuracy_std_dev": 0.007481944466909486, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.08116497474747475, "avg_duration_per_task_ms": 696165.8154623155, "avg_output_tokens_per_task": 59217.08663558664, "last_run_timestamp": "2026-10-31T16:05:36.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.832772, "accuracy_std_dev": 0.0698509274145066, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.024015700336700335, "avg_duration_per_task_ms": 613738.0185185185, "avg_output_tokens_per_task": 59848.74074074074, "last_run_timestamp": "2026-10-31T08:32:40.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8327310906791908, "accuracy_std_dev": 0.020914023298509846, "run_count": 14, "total_questions": 2768, "avg_cost_per_task_usd": 0.02508389053468208, "avg_duration_per_task_ms": 393221.2268786127, "avg_output_tokens_per_task": 46919.84176300578, "last_run_timestamp": "2026-10-31T08:25:10.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8326857692307692, "accuracy_std_dev": 0.011020358329744859, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03860720590520591, "avg_duration_per_task_ms": 230762.9052059052, "avg_output_tokens_per_task": 14476.024864024865, "last_run_timestamp": "2026-10-31T08:25:00.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8326600000000001, "accuracy_std_dev": 0.017376865045226092, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.21192213939393936, "avg_duration_per_task_ms": 312080.35656565655, "avg_output_tokens_per_task": 48262.278787878786, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8325688873598369, "accuracy_std_dev": 0.019237670701256954, "run_count": 10, "total_questions": 1962, "avg_cost_per_task_usd": 0.013576182466870541, "avg_duration_per_task_ms": 438297.6029561672, "avg_output_tokens_per_task": 39848.997961264016, "last_run_timestamp": "2026-10-31T08:26:26.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8325147922396857, "accuracy_std_dev": 0.029981271466154875, "run_count": 11, "total_questions": 2036, "avg_cost_per_task_usd": 0.011545964145383104, "avg_duration_per_task_ms": 399504.8664047151, "avg_output_tokens_per_task": 22873.502946954814, "last_run_timestamp": "2026-10-31T08:40:04.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.832407, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.05314901111111111, "avg_duration_per_task_ms": 479967.9111111111, "avg_output_tokens_per_task": 22823.577777777777, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8319864, "accuracy_std_dev": 0.01402072559605957, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06441977171717173, "avg_duration_per_task_ms": 117114.99292929293, "avg_output_tokens_per_task": 13401.907070707071, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8318478292335116, "accuracy_std_dev": 0.010811745465322452, "run_count": 17, "total_questions": 2805, "avg_cost_per_task_usd": 0.020570678431372547, "avg_duration_per_task_ms": 255956.05668449198, "avg_output_tokens_per_task": 40944.02673796791, "last_run_timestamp": "2026-10-31T08:20:07.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8317821610051186, "accuracy_std_dev": 0.013001907940092663, "run_count": 11, "total_questions": 2149, "avg_cost_per_task_usd": 0.15365638017682642, "avg_duration_per_task_ms": 272025.46998604, "avg_output_tokens_per_task": 34707.94276407632, "last_run_timestamp": "2026-11-05T00:00:09.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.831769061433447, "accuracy_std_dev": 0.006801018573632123, "run_count": 12, "total_questions": 2344, "avg_cost_per_task_usd": 0.03448967320819113, "avg_duration_per_task_ms": 208520.74530716724, "avg_output_tokens_per_task": 14194.120733788395, "last_run_timestamp": "2026-10-31T08:37:28.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.8316496, "accuracy_std_dev": 0.007062535765573171, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.022192562626262628, "avg_duration_per_task_ms": 11557.427272727273, "avg_output_tokens_per_task": 14346.162626262627, "last_run_timestamp": "2026-10-31T08:15:46.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8316432832699621, "accuracy_std_dev": 0.013089827300825316, "run_count": 9, "total_questions": 1578, "avg_cost_per_task_usd": 0.2110187237008872, "avg_duration_per_task_ms": 469662.1115335868, "avg_output_tokens_per_task": 47863.309252217994, "last_run_timestamp": "2026-10-31T16:35:08.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8315871731272918, "accuracy_std_dev": 0.020923668294302502, "run_count": 20, "total_questions": 3818, "avg_cost_per_task_usd": 0.04136215741225774, "avg_duration_per_task_ms": 440931.9447354636, "avg_output_tokens_per_task": 45362.3198009429, "last_run_timestamp": "2026-10-31T16:27:38.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "PrimeIntellect", "accuracy": 0.831532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 185, "avg_cost_per_task_usd": 0.01692656216216216, "avg_duration_per_task_ms": 560352.7027027027, "avg_output_tokens_per_task": 13578.437837837839, "last_run_timestamp": "2026-10-31T18:41:15.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8314150997442455, "accuracy_std_dev": 0.012107134010057875, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.02770090409207161, "avg_duration_per_task_ms": 583377.9558823529, "avg_output_tokens_per_task": 14708.68989769821, "last_run_timestamp": "2026-10-31T08:48:50.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8313539815914488, "accuracy_std_dev": 0.013525888763386546, "run_count": 9, "total_questions": 1684, "avg_cost_per_task_usd": 0.02395000890736342, "avg_duration_per_task_ms": 287524.2090261283, "avg_output_tokens_per_task": 47679.917458432305, "last_run_timestamp": "2026-10-31T08:55:40.953Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.8310883333333333, "accuracy_std_dev": 0.009654676391377534, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20155932323232326, "avg_duration_per_task_ms": 266665.86195286195, "avg_output_tokens_per_task": 37910.23063973064, "last_run_timestamp": "2026-10-31T08:09:31.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8310010869565219, "accuracy_std_dev": 0.01538004204178586, "run_count": 13, "total_questions": 2001, "avg_cost_per_task_usd": 0.04013055072463768, "avg_duration_per_task_ms": 299275.6486756622, "avg_output_tokens_per_task": 32699.943028485755, "last_run_timestamp": "2026-10-31T16:19:08.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8309762, "accuracy_std_dev": 0.004338052761320463, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04722357777777777, "avg_duration_per_task_ms": 258256.24646464645, "avg_output_tokens_per_task": 15298.06111111111, "last_run_timestamp": "2026-10-31T08:20:09.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8303971644295303, "accuracy_std_dev": 0.07664602971803865, "run_count": 13, "total_questions": 2384, "avg_cost_per_task_usd": 0.040612590604026846, "avg_duration_per_task_ms": 150268.53691275168, "avg_output_tokens_per_task": 19434.7072147651, "last_run_timestamp": "2026-10-31T16:17:39.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.830387, "accuracy_std_dev": 0.012044722890129136, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007987552398989899, "avg_duration_per_task_ms": 519162.2380050505, "avg_output_tokens_per_task": 20553.491792929293, "last_run_timestamp": "2026-10-31T16:01:53.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8302074978687126, "accuracy_std_dev": 0.018798024809619108, "run_count": 12, "total_questions": 2346, "avg_cost_per_task_usd": 0.14303728772378516, "avg_duration_per_task_ms": 291115.02642796247, "avg_output_tokens_per_task": 32292.15643648764, "last_run_timestamp": "2026-10-31T08:14:43.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8301065833333333, "accuracy_std_dev": 0.009803426675218673, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.06415462542087542, "avg_duration_per_task_ms": 540961.1531986533, "avg_output_tokens_per_task": 50401.976851851854, "last_run_timestamp": "2026-10-31T16:08:56.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.830042909090909, "accuracy_std_dev": 0.0271229167426385, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03839932506887053, "avg_duration_per_task_ms": 364543.0197428834, "avg_output_tokens_per_task": 38863.25987144169, "last_run_timestamp": "2026-10-24T16:01:48.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.8298383174258858, "accuracy_std_dev": 0.09656172773698192, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.08489557556037598, "avg_duration_per_task_ms": 562990.3246565438, "avg_output_tokens_per_task": 23490.788864786697, "last_run_timestamp": "2026-10-31T16:16:30.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.829787, "accuracy_std_dev": null, "run_count": 1, "total_questions": 141, "avg_cost_per_task_usd": 0.06982787943262411, "avg_duration_per_task_ms": 171508.63120567377, "avg_output_tokens_per_task": 17479.241134751774, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8295282648401827, "accuracy_std_dev": 0.02709499999999998, "run_count": 2, "total_questions": 219, "avg_cost_per_task_usd": 0.11806798630136987, "avg_duration_per_task_ms": 301952.6621004566, "avg_output_tokens_per_task": 26947.013698630137, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8290707212276216, "accuracy_std_dev": 0.048577604072121076, "run_count": 7, "total_questions": 1173, "avg_cost_per_task_usd": 0.04622304262574595, "avg_duration_per_task_ms": 675082.569479966, "avg_output_tokens_per_task": 30812.56862745098, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8290008259157149, "accuracy_std_dev": 0.02019136648766136, "run_count": 13, "total_questions": 2539, "avg_cost_per_task_usd": 0.25050771012209533, "avg_duration_per_task_ms": 410910.65537613234, "avg_output_tokens_per_task": 56752.72193777077, "last_run_timestamp": "2026-10-31T16:06:46.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8288813468715698, "accuracy_std_dev": 0.048221988340541345, "run_count": 14, "total_questions": 2733, "avg_cost_per_task_usd": 0.09670949213318698, "avg_duration_per_task_ms": 542922.2052689352, "avg_output_tokens_per_task": 29030.10171972192, "last_run_timestamp": "2026-10-31T08:36:47.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8288806316793894, "accuracy_std_dev": 0.011093878321949571, "run_count": 3, "total_questions": 524, "avg_cost_per_task_usd": 0.02635685687022901, "avg_duration_per_task_ms": 109838.5858778626, "avg_output_tokens_per_task": 20953.05534351145, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.828487042983565, "accuracy_std_dev": 0.04343161920830901, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.08886351622418878, "avg_duration_per_task_ms": 761181.4357353561, "avg_output_tokens_per_task": 31575.004214075012, "last_run_timestamp": "2026-10-31T16:01:53.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.82828275, "accuracy_std_dev": 0.004610363346581274, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0223316047979798, "avg_duration_per_task_ms": 615131.3409090909, "avg_output_tokens_per_task": 56159.15404040404, "last_run_timestamp": "2026-10-17T08:18:07.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8280725000000001, "accuracy_std_dev": 0.008005017801354371, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.047915450757575764, "avg_duration_per_task_ms": 452440.26767676766, "avg_output_tokens_per_task": 26534.96148989899, "last_run_timestamp": "2026-10-31T16:10:31.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.8280023333333334, "accuracy_std_dev": 0.01954028429225691, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11466727609427607, "avg_duration_per_task_ms": 496633.7373737374, "avg_output_tokens_per_task": 31920.87037037037, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8280021666666666, "accuracy_std_dev": 0.014721942482981734, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09966979545454545, "avg_duration_per_task_ms": 354652.75336700334, "avg_output_tokens_per_task": 45738.045454545456, "last_run_timestamp": "2026-10-17T16:06:16.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8273763076923075, "accuracy_std_dev": 0.07054214326135352, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.09363684304584305, "avg_duration_per_task_ms": 516457.86635586637, "avg_output_tokens_per_task": 34542.806915306915, "last_run_timestamp": "2026-10-31T16:05:36.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8270583636363636, "accuracy_std_dev": 0.00941702237510474, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.04288225068870524, "avg_duration_per_task_ms": 229942.66437098256, "avg_output_tokens_per_task": 14178.259871441689, "last_run_timestamp": "2026-10-31T08:36:47.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.82695, "accuracy_std_dev": 0.05592365383568327, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05177901725589225, "avg_duration_per_task_ms": 522861.78745791246, "avg_output_tokens_per_task": 24790.76473063973, "last_run_timestamp": "2026-10-31T16:01:53.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8266521780172412, "accuracy_std_dev": 0.022372414615840108, "run_count": 12, "total_questions": 2320, "avg_cost_per_task_usd": 0.02234062025862069, "avg_duration_per_task_ms": 235753.16637931034, "avg_output_tokens_per_task": 44456.286637931036, "last_run_timestamp": "2026-10-31T08:28:04.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Nebius", "accuracy": 0.8265989999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.051792560606060606, "avg_duration_per_task_ms": 150332.1313131313, "avg_output_tokens_per_task": 17173.429292929293, "last_run_timestamp": "2026-11-04T18:02:01.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.826459, "accuracy_std_dev": 0.009410002267091476, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.14383780639730645, "avg_duration_per_task_ms": 198190.3564814815, "avg_output_tokens_per_task": 35703.72769360269, "last_run_timestamp": "2026-10-31T08:32:01.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.826417643153527, "accuracy_std_dev": 0.01028202813164795, "run_count": 4, "total_questions": 723, "avg_cost_per_task_usd": 0.010974759336099586, "avg_duration_per_task_ms": 1541635.907330567, "avg_output_tokens_per_task": 31233.395573997233, "last_run_timestamp": "2026-10-31T08:24:48.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.8264159143968872, "accuracy_std_dev": 0.0040531378816294926, "run_count": 4, "total_questions": 771, "avg_cost_per_task_usd": 0.13822659403372245, "avg_duration_per_task_ms": 272716.0363164721, "avg_output_tokens_per_task": 34382.090791180286, "last_run_timestamp": "2026-10-31T08:38:15.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8264111325878595, "accuracy_std_dev": 0.01425677691801343, "run_count": 10, "total_questions": 1878, "avg_cost_per_task_usd": 0.02454355484558041, "avg_duration_per_task_ms": 264967.17305644305, "avg_output_tokens_per_task": 50263.019169329076, "last_run_timestamp": "2026-10-31T08:17:29.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8263836795903439, "accuracy_std_dev": 0.029166866420929173, "run_count": 7, "total_questions": 1367, "avg_cost_per_task_usd": 0.07172207900512069, "avg_duration_per_task_ms": 248028.12801755668, "avg_output_tokens_per_task": 21803.532553035846, "last_run_timestamp": "2026-10-31T09:17:12.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8259871818181819, "accuracy_std_dev": 0.010774204132752242, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03615010422405877, "avg_duration_per_task_ms": 305458.8122130395, "avg_output_tokens_per_task": 15557.78971533517, "last_run_timestamp": "2026-10-31T08:32:33.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8259259000000001, "accuracy_std_dev": 0.018259649079048597, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.012357968686868687, "avg_duration_per_task_ms": 759517.2151515152, "avg_output_tokens_per_task": 36563.7601010101, "last_run_timestamp": "2026-10-17T08:04:21.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.82575775, "accuracy_std_dev": 0.010065865980505628, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.14380135795454543, "avg_duration_per_task_ms": 418612.6521464646, "avg_output_tokens_per_task": 28784.420454545456, "last_run_timestamp": "2026-10-31T08:40:11.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8257574999999999, "accuracy_std_dev": 0.015993499999999994, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2609003181818182, "avg_duration_per_task_ms": 231557.84848484848, "avg_output_tokens_per_task": 25997.717171717173, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8255892, "accuracy_std_dev": 0.023390445078279278, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01099114494949495, "avg_duration_per_task_ms": 757385.2308080808, "avg_output_tokens_per_task": 39012.57828282828, "last_run_timestamp": "2026-10-17T08:02:36.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8254769999999999, "accuracy_std_dev": 0.021834969445059152, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025108806397306397, "avg_duration_per_task_ms": 234041.265993266, "avg_output_tokens_per_task": 21052.969696969696, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.825389385046729, "accuracy_std_dev": 0.014184335849948446, "run_count": 11, "total_questions": 2140, "avg_cost_per_task_usd": 0.03383954813084113, "avg_duration_per_task_ms": 407090.0654205608, "avg_output_tokens_per_task": 47991.5476635514, "last_run_timestamp": "2026-10-31T08:48:01.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8253217093954844, "accuracy_std_dev": 0.12574835204649074, "run_count": 7, "total_questions": 1373, "avg_cost_per_task_usd": 0.12495739184268023, "avg_duration_per_task_ms": 776035.8018936635, "avg_output_tokens_per_task": 37930.34668608886, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.82491575, "accuracy_std_dev": 0.013148603345507848, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19637407828282827, "avg_duration_per_task_ms": 945885.1603535353, "avg_output_tokens_per_task": 52273.792929292926, "last_run_timestamp": "2026-10-31T16:50:02.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8249156666666667, "accuracy_std_dev": 0.016206137938722134, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.024119345117845116, "avg_duration_per_task_ms": 456487.2491582492, "avg_output_tokens_per_task": 22656.301346801345, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8249156666666667, "accuracy_std_dev": 0.0254575838907693, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.07950963804713804, "avg_duration_per_task_ms": 692043.8686868687, "avg_output_tokens_per_task": 43016.01683501683, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.8249156249999999, "accuracy_std_dev": 0.030431425779683337, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09934535101010099, "avg_duration_per_task_ms": 1389882.110479798, "avg_output_tokens_per_task": 45786.19065656565, "last_run_timestamp": "2026-10-31T16:08:33.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.8243546666666667, "accuracy_std_dev": 0.03111308863734931, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.25956171212121215, "avg_duration_per_task_ms": 233681.0909090909, "avg_output_tokens_per_task": 25863.887205387204, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8243546666666666, "accuracy_std_dev": 0.03801456116045827, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.029119777356902358, "avg_duration_per_task_ms": 574036.2205387205, "avg_output_tokens_per_task": 53199.25168350169, "last_run_timestamp": "2026-11-01T11:18:30.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8242303292875989, "accuracy_std_dev": 0.016434035763013298, "run_count": 20, "total_questions": 3790, "avg_cost_per_task_usd": 0.030382969920844324, "avg_duration_per_task_ms": 266171.0530343008, "avg_output_tokens_per_task": 14053.306068601583, "last_run_timestamp": "2026-10-31T08:10:37.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.8241641919770775, "accuracy_std_dev": 0.013440726394979233, "run_count": 9, "total_questions": 1745, "avg_cost_per_task_usd": 0.01836141260744986, "avg_duration_per_task_ms": 609134.4246418338, "avg_output_tokens_per_task": 20155.189684813755, "last_run_timestamp": "2026-10-31T16:13:07.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8240738888888891, "accuracy_std_dev": 0.06341916550564268, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02869584455667789, "avg_duration_per_task_ms": 721909.4079685747, "avg_output_tokens_per_task": 30019.711560044892, "last_run_timestamp": "2026-10-17T08:18:04.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8240561561100449, "accuracy_std_dev": 0.00960574900775572, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.008232310300703776, "avg_duration_per_task_ms": 344296.5265515035, "avg_output_tokens_per_task": 19189.83877159309, "last_run_timestamp": "2026-10-31T16:08:31.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8237933333333333, "accuracy_std_dev": 0.003967811851498118, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01872473063973064, "avg_duration_per_task_ms": 507797.69696969696, "avg_output_tokens_per_task": 18599.063973063974, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8236530000000001, "accuracy_std_dev": 0.010804411622110653, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02350044696969697, "avg_duration_per_task_ms": 359036.5871212121, "avg_output_tokens_per_task": 20960.843434343435, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.82323225, "accuracy_std_dev": 0.029134592633800457, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.2227477537878788, "avg_duration_per_task_ms": 1217960.3686868686, "avg_output_tokens_per_task": 50933.75315656565, "last_run_timestamp": "2026-10-31T16:30:07.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8228234324639938, "accuracy_std_dev": 0.0330962900700723, "run_count": 13, "total_questions": 2569, "avg_cost_per_task_usd": 0.08174266757493187, "avg_duration_per_task_ms": 338894.74659400544, "avg_output_tokens_per_task": 20300.639159205915, "last_run_timestamp": "2026-10-31T16:02:14.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8226709999999999, "accuracy_std_dev": 0.023502234801538922, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02574937037037037, "avg_duration_per_task_ms": 439979.5740740741, "avg_output_tokens_per_task": 23407.67845117845, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8226336142278774, "accuracy_std_dev": 0.01129717437085189, "run_count": 9, "total_questions": 1729, "avg_cost_per_task_usd": 0.02482087044534413, "avg_duration_per_task_ms": 433623.25274725276, "avg_output_tokens_per_task": 24707.683632157317, "last_run_timestamp": "2026-10-31T16:42:08.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8225450489150423, "accuracy_std_dev": 0.023532795309234442, "run_count": 14, "total_questions": 2719, "avg_cost_per_task_usd": 0.026525477749172497, "avg_duration_per_task_ms": 850874.7859507172, "avg_output_tokens_per_task": 49891.477013607946, "last_run_timestamp": "2026-10-31T08:42:18.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8223905, "accuracy_std_dev": 0.019262412208322532, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03192426473063974, "avg_duration_per_task_ms": 260830.34259259258, "avg_output_tokens_per_task": 14583.287878787878, "last_run_timestamp": "2026-10-31T08:17:29.187Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.8215486666666668, "accuracy_std_dev": 0.015611992961253275, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02833054377104377, "avg_duration_per_task_ms": 228888.5606060606, "avg_output_tokens_per_task": 56421.28451178451, "last_run_timestamp": "2026-10-31T08:03:09.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8207070000000001, "accuracy_std_dev": 0.024411000000000016, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04063828030303031, "avg_duration_per_task_ms": 83296.42424242424, "avg_output_tokens_per_task": 26969.131313131315, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8204965000000001, "accuracy_std_dev": 0.019212136906133062, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0063976704545454545, "avg_duration_per_task_ms": 423192.5814393939, "avg_output_tokens_per_task": 18733.522727272728, "last_run_timestamp": "2026-10-31T16:12:28.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8203833076923077, "accuracy_std_dev": 0.018076297388844354, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.042996285159285164, "avg_duration_per_task_ms": 241748.8038073038, "avg_output_tokens_per_task": 14318.661227661227, "last_run_timestamp": "2026-10-31T08:09:30.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.820286125, "accuracy_std_dev": 0.014129067020131756, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007522392676767676, "avg_duration_per_task_ms": 484868.283459596, "avg_output_tokens_per_task": 18453.40404040404, "last_run_timestamp": "2026-10-31T16:13:09.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.8202743410372041, "accuracy_std_dev": 0.011484132116496031, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.09063781341600902, "avg_duration_per_task_ms": 168767.6234498309, "avg_output_tokens_per_task": 22236.227170236754, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.8200275393258427, "accuracy_std_dev": 0.011794769394173345, "run_count": 9, "total_questions": 1691, "avg_cost_per_task_usd": 0.03818077646363098, "avg_duration_per_task_ms": 215799.96629213484, "avg_output_tokens_per_task": 14692.68894145476, "last_run_timestamp": "2026-10-31T08:55:11.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8194060909090909, "accuracy_std_dev": 0.013884218840464785, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.033594433884297524, "avg_duration_per_task_ms": 211994.46005509642, "avg_output_tokens_per_task": 13897.44214876033, "last_run_timestamp": "2026-10-31T08:09:30.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8192149207920791, "accuracy_std_dev": 0.018823182524040232, "run_count": 6, "total_questions": 909, "avg_cost_per_task_usd": 0.11224348184818483, "avg_duration_per_task_ms": 401079.6094609461, "avg_output_tokens_per_task": 46532.80198019802, "last_run_timestamp": "2026-10-17T16:06:46.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.8188040208464941, "accuracy_std_dev": 0.005162697447798004, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.02620403032217309, "avg_duration_per_task_ms": 318291.60138976626, "avg_output_tokens_per_task": 26132.884396715097, "last_run_timestamp": "2026-10-31T16:04:48.178Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8186413984710942, "accuracy_std_dev": 0.062429697723794066, "run_count": 22, "total_questions": 4186, "avg_cost_per_task_usd": 0.0077469773053033915, "avg_duration_per_task_ms": 504549.9739608218, "avg_output_tokens_per_task": 17360.977544194935, "last_run_timestamp": "2026-10-31T16:45:45.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.8180615714285715, "accuracy_std_dev": 0.014803209766390367, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015330031024531026, "avg_duration_per_task_ms": 451421.6443001443, "avg_output_tokens_per_task": 12902.48051948052, "last_run_timestamp": "2026-11-04T17:56:05.778Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.8178727706422019, "accuracy_std_dev": 0.03174962566960436, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.023857419979612642, "avg_duration_per_task_ms": 166236.06523955148, "avg_output_tokens_per_task": 14523.781855249745, "last_run_timestamp": "2026-10-31T08:07:51.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.81776075, "accuracy_std_dev": 0.030405091048827666, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.08916797159090908, "avg_duration_per_task_ms": 1052180.1603535353, "avg_output_tokens_per_task": 43838.132575757576, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8177376144306652, "accuracy_std_dev": 0.03151916867911054, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.013423886696730551, "avg_duration_per_task_ms": 855848.4718151072, "avg_output_tokens_per_task": 39679.556933483655, "last_run_timestamp": "2026-10-17T08:19:03.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.8173077377622378, "accuracy_std_dev": 0.04183386114142466, "run_count": 3, "total_questions": 572, "avg_cost_per_task_usd": 0.00918055944055944, "avg_duration_per_task_ms": 556054.2482517483, "avg_output_tokens_per_task": 20397.216783216783, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8172424786585366, "accuracy_std_dev": 0.032381379178472315, "run_count": 8, "total_questions": 984, "avg_cost_per_task_usd": 0.0567359319105691, "avg_duration_per_task_ms": 328244.918699187, "avg_output_tokens_per_task": 21710.94918699187, "last_run_timestamp": "2026-10-31T16:26:17.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8171817336142323, "accuracy_std_dev": 0.04982830145510509, "run_count": 11, "total_questions": 2136, "avg_cost_per_task_usd": 0.018343408239700373, "avg_duration_per_task_ms": 793559.7439138577, "avg_output_tokens_per_task": 48776.84925093633, "last_run_timestamp": "2026-10-31T08:31:53.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8171795172255397, "accuracy_std_dev": 0.008683223783407119, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.03297541662838769, "avg_duration_per_task_ms": 545292.2503445108, "avg_output_tokens_per_task": 15758.780431786863, "last_run_timestamp": "2026-10-31T08:29:48.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8170593333333332, "accuracy_std_dev": 0.013208478749988154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.018857220538720538, "avg_duration_per_task_ms": 518487.1228956229, "avg_output_tokens_per_task": 18659.164983164985, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8170593333333332, "accuracy_std_dev": 0.004418546090690392, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.006106776094276093, "avg_duration_per_task_ms": 508955.7373737374, "avg_output_tokens_per_task": 18545.604377104377, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.8170593333333332, "accuracy_std_dev": 0.006780464700561124, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.08866408754208754, "avg_duration_per_task_ms": 189591.79966329967, "avg_output_tokens_per_task": 25298.508417508416, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8169191666666666, "accuracy_std_dev": 0.011358349086562799, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.3136177163299663, "avg_duration_per_task_ms": 289623.9978956229, "avg_output_tokens_per_task": 20717.387205387204, "last_run_timestamp": "2026-10-31T08:15:24.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.8169191250000001, "accuracy_std_dev": 0.047316166086332226, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.20553683143939394, "avg_duration_per_task_ms": 898204.4223484849, "avg_output_tokens_per_task": 51369.868055555555, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.8168020459873759, "accuracy_std_dev": 0.03036523422023943, "run_count": 6, "total_questions": 1109, "avg_cost_per_task_usd": 0.028931809738503157, "avg_duration_per_task_ms": 508171.53381424706, "avg_output_tokens_per_task": 17143.577998196575, "last_run_timestamp": "2026-10-17T16:03:44.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.81649825, "accuracy_std_dev": 0.007896181684048294, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.026513486111111113, "avg_duration_per_task_ms": 167722.74494949495, "avg_output_tokens_per_task": 23391.1047979798, "last_run_timestamp": "2026-10-31T16:09:44.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.815937, "accuracy_std_dev": 0.00915257617650169, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06866670033670035, "avg_duration_per_task_ms": 839163.7121212122, "avg_output_tokens_per_task": 39474.887205387204, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8152402988505746, "accuracy_std_dev": 0.029604949040447602, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.28207722477650066, "avg_duration_per_task_ms": 913648.006385696, "avg_output_tokens_per_task": 58474.00127713921, "last_run_timestamp": "2026-10-31T16:12:07.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8150252499999998, "accuracy_std_dev": 0.004075243451316723, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0115347702020202, "avg_duration_per_task_ms": 842527.0176767677, "avg_output_tokens_per_task": 18589.04419191919, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.8148633940983607, "accuracy_std_dev": 0.008422212201041645, "run_count": 8, "total_questions": 1525, "avg_cost_per_task_usd": 0.042689679344262293, "avg_duration_per_task_ms": 185447.63540983608, "avg_output_tokens_per_task": 16898.064918032785, "last_run_timestamp": "2026-11-04T18:19:31.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8148148, "accuracy_std_dev": 0.009606302751839527, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.33180516727272724, "avg_duration_per_task_ms": 293289.88505050505, "avg_output_tokens_per_task": 20920.7098989899, "last_run_timestamp": "2026-10-31T08:27:57.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8146744166666667, "accuracy_std_dev": 0.010590848930864255, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.36361028135521894, "avg_duration_per_task_ms": 357888.61384680134, "avg_output_tokens_per_task": 22899.654882154882, "last_run_timestamp": "2026-10-31T08:33:07.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.813973, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04186610353535354, "avg_duration_per_task_ms": 76135.87878787878, "avg_output_tokens_per_task": 27786.689393939392, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.8136124285714287, "accuracy_std_dev": 0.04335004526231662, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14883965584415582, "avg_duration_per_task_ms": 1660514.1673881675, "avg_output_tokens_per_task": 48710.61111111111, "last_run_timestamp": "2026-10-31T16:08:56.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8133718571428571, "accuracy_std_dev": 0.016208212586997554, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04426409668109668, "avg_duration_per_task_ms": 84524.13997113997, "avg_output_tokens_per_task": 27404.910533910534, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0701835909090909, "avg_duration_per_task_ms": 493979.91414141416, "avg_output_tokens_per_task": 29227.76767676768, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.813051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.00766168253968254, "avg_duration_per_task_ms": 356938.3703703704, "avg_output_tokens_per_task": 33904.37566137566, "last_run_timestamp": "2026-10-14T18:00:03.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.811448, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1522642575757576, "avg_duration_per_task_ms": 39952.71717171717, "avg_output_tokens_per_task": 2844.2373737373737, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.810606, "accuracy_std_dev": 0.010546997511140292, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.044063582702020196, "avg_duration_per_task_ms": 62743.707702020205, "avg_output_tokens_per_task": 15119.022095959595, "last_run_timestamp": "2026-10-31T08:43:59.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8104117692307693, "accuracy_std_dev": 0.017452206376509655, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.3396733146853147, "avg_duration_per_task_ms": 352646.4836829837, "avg_output_tokens_per_task": 22475.542735042734, "last_run_timestamp": "2026-10-31T08:46:10.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8103954999999999, "accuracy_std_dev": 0.025710588553162297, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.009251354166666666, "avg_duration_per_task_ms": 588683.1515151515, "avg_output_tokens_per_task": 19069.81186868687, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8101851666666666, "accuracy_std_dev": 0.020562625427513432, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.3433783051346802, "avg_duration_per_task_ms": 356918.0021043771, "avg_output_tokens_per_task": 22728.853956228955, "last_run_timestamp": "2026-10-31T08:17:29.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8099512222222223, "accuracy_std_dev": 0.024479317441363887, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11924308024691359, "avg_duration_per_task_ms": 781765.5089786757, "avg_output_tokens_per_task": 62033.351851851854, "last_run_timestamp": "2026-10-31T16:10:09.226Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.8097645, "accuracy_std_dev": 0.021885500000000002, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044968992424242425, "avg_duration_per_task_ms": 208495.61616161617, "avg_output_tokens_per_task": 22388.290404040403, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Chutes", "accuracy": 0.8097639999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06257432828282827, "avg_duration_per_task_ms": 366659.297979798, "avg_output_tokens_per_task": 47164.93434343435, "last_run_timestamp": "2026-11-01T18:00:07.322Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.8096608426103648, "accuracy_std_dev": 0.012817931849127972, "run_count": 3, "total_questions": 521, "avg_cost_per_task_usd": 0.0079749443378119, "avg_duration_per_task_ms": 289711.4932821497, "avg_output_tokens_per_task": 31380.54894433781, "last_run_timestamp": "2026-10-31T16:04:23.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.8089722206946454, "accuracy_std_dev": 0.07286502949135142, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.1989648010130246, "avg_duration_per_task_ms": 1711843.8125904487, "avg_output_tokens_per_task": 46110.4507959479, "last_run_timestamp": "2026-10-31T16:22:14.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8089225, "accuracy_std_dev": 0.06617775297824792, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09753928787878788, "avg_duration_per_task_ms": 791029.2310606061, "avg_output_tokens_per_task": 39011.65909090909, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8081858749999998, "accuracy_std_dev": 0.01700473100667502, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.23625982323232325, "avg_duration_per_task_ms": 1060789.2386363635, "avg_output_tokens_per_task": 63306.07323232323, "last_run_timestamp": "2026-10-31T16:12:28.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.8080808, "accuracy_std_dev": 0.053692861534472135, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.2018329616161616, "avg_duration_per_task_ms": 1561759.3434343433, "avg_output_tokens_per_task": 48030.74141414141, "last_run_timestamp": "2026-10-31T16:31:54.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.807532574733096, "accuracy_std_dev": 0.023704551349195916, "run_count": 6, "total_questions": 1124, "avg_cost_per_task_usd": 0.039211227758007124, "avg_duration_per_task_ms": 252786.0587188612, "avg_output_tokens_per_task": 39158.109430604985, "last_run_timestamp": "2026-10-31T16:09:44.822Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.80723875, "accuracy_std_dev": 0.007668707465244688, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02855746717171717, "avg_duration_per_task_ms": 256654.82196969696, "avg_output_tokens_per_task": 56910.10353535353, "last_run_timestamp": "2026-11-03T01:57:50.570Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8069586666666666, "accuracy_std_dev": 0.03732496637491973, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026121195286195286, "avg_duration_per_task_ms": 474388.5303030303, "avg_output_tokens_per_task": 21767.089225589225, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.8068179999999999, "accuracy_std_dev": 0.027363478050131, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.046258910353535354, "avg_duration_per_task_ms": 196125.59974747474, "avg_output_tokens_per_task": 21946.10606060606, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.806397, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006770893939393939, "avg_duration_per_task_ms": 1110169.2828282828, "avg_output_tokens_per_task": 17470.782828282827, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.806397, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04417995707070707, "avg_duration_per_task_ms": 210130.46464646465, "avg_output_tokens_per_task": 21993.772727272728, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.805275, "accuracy_std_dev": 0.018459119444509446, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.10121038888888889, "avg_duration_per_task_ms": 584553.9983164984, "avg_output_tokens_per_task": 37644.86531986532, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.80492425, "accuracy_std_dev": 0.025086207414384096, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.2853950606060606, "avg_duration_per_task_ms": 994351.720959596, "avg_output_tokens_per_task": 64899.24684343435, "last_run_timestamp": "2026-10-31T16:12:28.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.804281, "accuracy_std_dev": null, "run_count": 1, "total_questions": 109, "avg_cost_per_task_usd": 0.08876922935779817, "avg_duration_per_task_ms": 124213.02752293578, "avg_output_tokens_per_task": 36876.293577981654, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8041526666666666, "accuracy_std_dev": 0.02864721697167495, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.028751296296296296, "avg_duration_per_task_ms": 459929.2138047138, "avg_output_tokens_per_task": 25017.868686868685, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.8040123333333333, "accuracy_std_dev": 0.026646777658012516, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.029773004208754207, "avg_duration_per_task_ms": 321073.38468013465, "avg_output_tokens_per_task": 19724.395622895623, "last_run_timestamp": "2026-10-17T16:09:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8038719166666666, "accuracy_std_dev": 0.01983021612530708, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.3387011611952862, "avg_duration_per_task_ms": 355211.85016835015, "avg_output_tokens_per_task": 22440.643097643097, "last_run_timestamp": "2026-10-31T08:53:23.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.8028182609489051, "accuracy_std_dev": 0.07654548992882891, "run_count": 9, "total_questions": 1644, "avg_cost_per_task_usd": 0.019102108272506083, "avg_duration_per_task_ms": 309069.2311435523, "avg_output_tokens_per_task": 40420.16605839416, "last_run_timestamp": "2026-10-31T08:33:53.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.8015174199706314, "accuracy_std_dev": 0.034082802627718296, "run_count": 10, "total_questions": 1362, "avg_cost_per_task_usd": 0.006941986784140968, "avg_duration_per_task_ms": 316770.08076358296, "avg_output_tokens_per_task": 27549.08516886931, "last_run_timestamp": "2026-10-31T16:04:29.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.801346625, "accuracy_std_dev": 0.04197794992593582, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.026733222222222217, "avg_duration_per_task_ms": 301102.7582070707, "avg_output_tokens_per_task": 25848.362373737375, "last_run_timestamp": "2026-10-31T16:05:36.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.8006073724696356, "accuracy_std_dev": 0.018676167604730915, "run_count": 5, "total_questions": 988, "avg_cost_per_task_usd": 0.026427639676113363, "avg_duration_per_task_ms": 228773.67307692306, "avg_output_tokens_per_task": 52620.16093117409, "last_run_timestamp": "2026-10-31T08:22:47.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.800084125, "accuracy_std_dev": 0.03890081182455419, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.18029773989898992, "avg_duration_per_task_ms": 560473.1414141414, "avg_output_tokens_per_task": 56128.871212121216, "last_run_timestamp": "2026-10-31T16:15:54.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7999998975308643, "accuracy_std_dev": 0.02657019067436704, "run_count": 6, "total_questions": 810, "avg_cost_per_task_usd": 0.06976524074074074, "avg_duration_per_task_ms": 1694220.9679012345, "avg_output_tokens_per_task": 39748.25802469136, "last_run_timestamp": "2026-10-31T16:05:14.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.79966325, "accuracy_std_dev": 0.00518913881944781, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013944108585858586, "avg_duration_per_task_ms": 338268.0871212121, "avg_output_tokens_per_task": 14269.474747474747, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.7979796666666668, "accuracy_std_dev": 0.06576415746353702, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.041038570707070704, "avg_duration_per_task_ms": 1120543.265993266, "avg_output_tokens_per_task": 23360.329966329966, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7974186666666667, "accuracy_std_dev": 0.009356358206529358, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014677102693602694, "avg_duration_per_task_ms": 310917.8265993266, "avg_output_tokens_per_task": 14041.452861952863, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.79706775, "accuracy_std_dev": 0.041385108447212035, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.043133531565656566, "avg_duration_per_task_ms": 497051.0765993266, "avg_output_tokens_per_task": 42719.71127946128, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7965251890459364, "accuracy_std_dev": 0.006229073430740378, "run_count": 6, "total_questions": 1132, "avg_cost_per_task_usd": 0.05014427738515901, "avg_duration_per_task_ms": 1091376.4240282685, "avg_output_tokens_per_task": 22832.588339222613, "last_run_timestamp": "2026-10-17T16:06:10.469Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7955151838886546, "accuracy_std_dev": 0.021167952217358864, "run_count": 12, "total_questions": 2371, "avg_cost_per_task_usd": 0.0645802066638549, "avg_duration_per_task_ms": 194462.45592576973, "avg_output_tokens_per_task": 20212.99072121468, "last_run_timestamp": "2026-10-31T08:22:42.983Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7940516666666667, "accuracy_std_dev": 0.017513670362192944, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17459135353535354, "avg_duration_per_task_ms": 780357.1144781145, "avg_output_tokens_per_task": 55892.003367003366, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7940513333333334, "accuracy_std_dev": 0.009654676391377534, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013367496632996632, "avg_duration_per_task_ms": 287892.37373737374, "avg_output_tokens_per_task": 13942.378787878788, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7937709000000001, "accuracy_std_dev": 0.011766387869265602, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.004587615656565657, "avg_duration_per_task_ms": 253050.72525252524, "avg_output_tokens_per_task": 30034.337373737373, "last_run_timestamp": "2026-10-17T16:03:34.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.7931395, "accuracy_std_dev": 0.031746818911506706, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03944074053030303, "avg_duration_per_task_ms": 195523.23863636365, "avg_output_tokens_per_task": 21811.546085858587, "last_run_timestamp": "2026-10-31T16:08:31.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7926977023289665, "accuracy_std_dev": 0.008002668144348565, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.0120994730713246, "avg_duration_per_task_ms": 824903.4272197962, "avg_output_tokens_per_task": 39852.10480349345, "last_run_timestamp": "2026-11-04T19:16:42.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7925247381378694, "accuracy_std_dev": 0.015422197277112174, "run_count": 12, "total_questions": 2234, "avg_cost_per_task_usd": 0.004671950760966876, "avg_duration_per_task_ms": 1479604.6521933752, "avg_output_tokens_per_task": 30665.252014324084, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7923760646992054, "accuracy_std_dev": 0.150042241777375, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.015620138479001138, "avg_duration_per_task_ms": 554789.2985244041, "avg_output_tokens_per_task": 26201.363223609536, "last_run_timestamp": "2026-10-17T08:10:43.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7920900579268293, "accuracy_std_dev": 0.011512971285033247, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.01824498475609756, "avg_duration_per_task_ms": 181543.2469512195, "avg_output_tokens_per_task": 15051.83943089431, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7916197777777776, "accuracy_std_dev": 0.10449398651148174, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.029375244107744107, "avg_duration_per_task_ms": 135177.0364758698, "avg_output_tokens_per_task": 22499.014029180697, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.7912454999999999, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05368662373737374, "avg_duration_per_task_ms": 408830.87373737374, "avg_output_tokens_per_task": 23323.171717171717, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.79103525, "accuracy_std_dev": 0.01161593222421257, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01797352335858586, "avg_duration_per_task_ms": 60686.5625, "avg_output_tokens_per_task": 5836.820707070707, "last_run_timestamp": "2026-10-31T16:10:36.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7907960444642064, "accuracy_std_dev": 0.011335818096181775, "run_count": 12, "total_questions": 2249, "avg_cost_per_task_usd": 0.008774971098265897, "avg_duration_per_task_ms": 377108.3983992886, "avg_output_tokens_per_task": 34581.72787905736, "last_run_timestamp": "2026-10-31T16:15:26.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.787879, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01368820202020202, "avg_duration_per_task_ms": 414867.4494949495, "avg_output_tokens_per_task": 14233.494949494949, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7876197692307692, "accuracy_std_dev": 0.03230095147242056, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.10465196542346543, "avg_duration_per_task_ms": 367907.6076146076, "avg_output_tokens_per_task": 27503.11655011655, "last_run_timestamp": "2026-10-31T16:09:41.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.7872267663934426, "accuracy_std_dev": 0.013963506030602306, "run_count": 3, "total_questions": 488, "avg_cost_per_task_usd": 0.008632290983606556, "avg_duration_per_task_ms": 586832.4200819673, "avg_output_tokens_per_task": 37091.375, "last_run_timestamp": "2026-10-31T16:20:33.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7871492666666666, "accuracy_std_dev": 0.11453375331110427, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.01689483905723906, "avg_duration_per_task_ms": 609264.4754208755, "avg_output_tokens_per_task": 30856.628282828282, "last_run_timestamp": "2026-11-06T22:10:48.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7868265000000001, "accuracy_std_dev": 0.010545056341717679, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.021032600378787878, "avg_duration_per_task_ms": 242572.1452020202, "avg_output_tokens_per_task": 12501.890782828283, "last_run_timestamp": "2026-10-31T16:10:36.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7865083595251555, "accuracy_std_dev": 0.02939069554623296, "run_count": 9, "total_questions": 1769, "avg_cost_per_task_usd": 0.011274476540418316, "avg_duration_per_task_ms": 619482.3962690786, "avg_output_tokens_per_task": 37337.42283776145, "last_run_timestamp": "2026-10-31T16:03:21.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7864661884870098, "accuracy_std_dev": 0.10678677266478277, "run_count": 10, "total_questions": 1963, "avg_cost_per_task_usd": 0.12194484309730003, "avg_duration_per_task_ms": 745901.2404482934, "avg_output_tokens_per_task": 33847.68874172185, "last_run_timestamp": "2026-10-31T17:00:44.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.786195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2643429696969697, "avg_duration_per_task_ms": 180778.88888888888, "avg_output_tokens_per_task": 17428.555555555555, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7861940813859789, "accuracy_std_dev": 0.020955855485826815, "run_count": 7, "total_questions": 1241, "avg_cost_per_task_usd": 0.04030405640612409, "avg_duration_per_task_ms": 737558.6027397261, "avg_output_tokens_per_task": 18622.764705882353, "last_run_timestamp": "2026-10-17T16:06:15.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7854701461538461, "accuracy_std_dev": 0.11062032174825517, "run_count": 4, "total_questions": 780, "avg_cost_per_task_usd": 0.10532577051282052, "avg_duration_per_task_ms": 1185336.45, "avg_output_tokens_per_task": 37894.10512820513, "last_run_timestamp": "2026-10-24T16:13:05.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.7854607925531915, "accuracy_std_dev": 0.004311000000000009, "run_count": 2, "total_questions": 376, "avg_cost_per_task_usd": 0.05007911170212766, "avg_duration_per_task_ms": 706928.4574468085, "avg_output_tokens_per_task": 21703.061170212764, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.78493275, "accuracy_std_dev": 0.010934561842501992, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03407076388888889, "avg_duration_per_task_ms": 483005.77398989897, "avg_output_tokens_per_task": 67906.58964646465, "last_run_timestamp": "2026-10-31T08:31:13.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7847687086124402, "accuracy_std_dev": 0.12803182926015888, "run_count": 11, "total_questions": 2090, "avg_cost_per_task_usd": 0.10580474545454546, "avg_duration_per_task_ms": 663710.6894736842, "avg_output_tokens_per_task": 30101.89138755981, "last_run_timestamp": "2026-10-31T16:40:50.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7846955057471265, "accuracy_std_dev": 0.10720142439753014, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.07534132247765006, "avg_duration_per_task_ms": 702935.9335887611, "avg_output_tokens_per_task": 27301.47062579821, "last_run_timestamp": "2026-10-10T16:29:40.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.7828283333333333, "accuracy_std_dev": 0.014481820841623765, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02915533333333333, "avg_duration_per_task_ms": 454449.42087542085, "avg_output_tokens_per_task": 24277.760942760942, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7825877142857144, "accuracy_std_dev": 0.026746482528225146, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.053754077200577205, "avg_duration_per_task_ms": 455374.8766233766, "avg_output_tokens_per_task": 23369.723665223664, "last_run_timestamp": "2026-10-17T16:03:34.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7807207329462988, "accuracy_std_dev": 0.06482919923639853, "run_count": 7, "total_questions": 1378, "avg_cost_per_task_usd": 0.00722692815674891, "avg_duration_per_task_ms": 850153.4637155298, "avg_output_tokens_per_task": 18320.05587808418, "last_run_timestamp": "2026-10-24T16:21:54.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7804782183266932, "accuracy_std_dev": 0.017482235460356838, "run_count": 9, "total_questions": 1255, "avg_cost_per_task_usd": 0.04653852270916335, "avg_duration_per_task_ms": 966873.6972111554, "avg_output_tokens_per_task": 23672.268525896416, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7800223333333333, "accuracy_std_dev": 0.011195005920300127, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.006137688552188552, "avg_duration_per_task_ms": 345084.8249158249, "avg_output_tokens_per_task": 14881.631313131313, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.77988225, "accuracy_std_dev": 0.01598762236661534, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3256477702020202, "avg_duration_per_task_ms": 1001085.1660353536, "avg_output_tokens_per_task": 82083.53661616161, "last_run_timestamp": "2026-10-31T16:04:02.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7797533759071117, "accuracy_std_dev": 0.019631960574532543, "run_count": 7, "total_questions": 1378, "avg_cost_per_task_usd": 0.02252375471698113, "avg_duration_per_task_ms": 854891.6850507982, "avg_output_tokens_per_task": 38423.74455732946, "last_run_timestamp": "2026-10-24T16:37:21.981Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.779461111111111, "accuracy_std_dev": 0.13454197858284844, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.008024448933782269, "avg_duration_per_task_ms": 403581.7491582492, "avg_output_tokens_per_task": 25395.133557800225, "last_run_timestamp": "2026-10-31T16:19:28.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "PrimeIntellect", "accuracy": 0.77933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 179, "avg_cost_per_task_usd": 0.00797877653631285, "avg_duration_per_task_ms": 499606.7541899441, "avg_output_tokens_per_task": 38804.5530726257, "last_run_timestamp": "2026-11-01T18:00:07.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.77925075, "accuracy_std_dev": 0.01980242038710168, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02103724936868687, "avg_duration_per_task_ms": 475607.264520202, "avg_output_tokens_per_task": 27969.91224747475, "last_run_timestamp": "2026-10-31T16:16:35.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7784991428571428, "accuracy_std_dev": 0.04711966006252765, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.013313156565656565, "avg_duration_per_task_ms": 825301.5577200577, "avg_output_tokens_per_task": 39367.342712842714, "last_run_timestamp": "2026-10-17T08:18:06.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.776936, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013052893939393939, "avg_duration_per_task_ms": 64108.87373737374, "avg_output_tokens_per_task": 10319.823232323231, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.7760940000000002, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013424281144781143, "avg_duration_per_task_ms": 64995.4595959596, "avg_output_tokens_per_task": 10615.195286195287, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7752525, "accuracy_std_dev": 0.02576620138055538, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011368148148148148, "avg_duration_per_task_ms": 910071.1422558923, "avg_output_tokens_per_task": 43361.55303030303, "last_run_timestamp": "2026-10-31T09:00:40.586Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7749718333333332, "accuracy_std_dev": 0.019112577010236546, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.005948839646464647, "avg_duration_per_task_ms": 683117.8017676767, "avg_output_tokens_per_task": 37109.85984848485, "last_run_timestamp": "2026-10-31T16:03:21.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7744107142857142, "accuracy_std_dev": 0.007947545348170354, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005179176046176046, "avg_duration_per_task_ms": 801836.0988455989, "avg_output_tokens_per_task": 28610.891053391053, "last_run_timestamp": "2026-10-31T16:17:18.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7741524627118646, "accuracy_std_dev": 0.024920321732780762, "run_count": 7, "total_questions": 1180, "avg_cost_per_task_usd": 0.010626239830508475, "avg_duration_per_task_ms": 1250036.8076271187, "avg_output_tokens_per_task": 40411.0593220339, "last_run_timestamp": "2026-10-31T08:17:43.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.77381, "accuracy_std_dev": null, "run_count": 1, "total_questions": 140, "avg_cost_per_task_usd": 0.00325335, "avg_duration_per_task_ms": 70786.31428571428, "avg_output_tokens_per_task": 3951.1785714285716, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7737795, "accuracy_std_dev": 0.054440706080101486, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15974002967171716, "avg_duration_per_task_ms": 914205.7670454546, "avg_output_tokens_per_task": 68299.18939393939, "last_run_timestamp": "2026-10-31T16:04:51.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7736031459537571, "accuracy_std_dev": 0.07146005981537049, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.01837689523121387, "avg_duration_per_task_ms": 1270075.3273121386, "avg_output_tokens_per_task": 17747.31430635838, "last_run_timestamp": "2026-10-31T16:45:39.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.7734006, "accuracy_std_dev": 0.005280843894681953, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014113778787878787, "avg_duration_per_task_ms": 123713.42222222222, "avg_output_tokens_per_task": 10542.545454545454, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.7732982293193716, "accuracy_std_dev": 0.013698592432801259, "run_count": 5, "total_questions": 955, "avg_cost_per_task_usd": 0.00232973612565445, "avg_duration_per_task_ms": 792784.167539267, "avg_output_tokens_per_task": 15348.367539267016, "last_run_timestamp": "2026-10-31T16:10:36.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7732734397522524, "accuracy_std_dev": 0.007399312031750627, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.011318120495495496, "avg_duration_per_task_ms": 924768.2730855856, "avg_output_tokens_per_task": 11765.394144144144, "last_run_timestamp": "2026-10-31T16:02:14.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7727273, "accuracy_std_dev": 0.04359568158671222, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.22926952626262626, "avg_duration_per_task_ms": 887813.6904040404, "avg_output_tokens_per_task": 76915.14747474747, "last_run_timestamp": "2026-10-17T16:06:46.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7722982392694063, "accuracy_std_dev": 0.026492001321153518, "run_count": 8, "total_questions": 1095, "avg_cost_per_task_usd": 0.013083080365296807, "avg_duration_per_task_ms": 53025.53789954338, "avg_output_tokens_per_task": 17342.828310502282, "last_run_timestamp": "2026-10-31T16:39:49.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7721661666666666, "accuracy_std_dev": 0.007120472442580984, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.020572006734006738, "avg_duration_per_task_ms": 737897.6776094276, "avg_output_tokens_per_task": 51210.5968013468, "last_run_timestamp": "2026-10-31T16:23:19.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.7714860192771085, "accuracy_std_dev": 0.016470310033005864, "run_count": 6, "total_questions": 830, "avg_cost_per_task_usd": 0.009284253012048193, "avg_duration_per_task_ms": 720483.7698795181, "avg_output_tokens_per_task": 30820.519277108433, "last_run_timestamp": "2026-10-31T16:24:40.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7714366132879046, "accuracy_std_dev": 0.005025261541010102, "run_count": 3, "total_questions": 587, "avg_cost_per_task_usd": 0.00432709880749574, "avg_duration_per_task_ms": 4109.833049403748, "avg_output_tokens_per_task": 2384.9233390119252, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7713804000000001, "accuracy_std_dev": 0.01747606822600553, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014499158585858585, "avg_duration_per_task_ms": 1080917.518181818, "avg_output_tokens_per_task": 51478.64141414141, "last_run_timestamp": "2026-10-31T08:09:31.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7706375705329154, "accuracy_std_dev": 0.012487370499829025, "run_count": 3, "total_questions": 319, "avg_cost_per_task_usd": 0.03633863949843261, "avg_duration_per_task_ms": 130589.2789968652, "avg_output_tokens_per_task": 37748.849529780564, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7705974206349206, "accuracy_std_dev": 0.01526894876684195, "run_count": 9, "total_questions": 1764, "avg_cost_per_task_usd": 0.021992218253968247, "avg_duration_per_task_ms": 244131.76473922902, "avg_output_tokens_per_task": 29949.219387755104, "last_run_timestamp": "2026-11-05T00:00:07.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7704452803915377, "accuracy_std_dev": 0.015743859747037087, "run_count": 16, "total_questions": 3167, "avg_cost_per_task_usd": 0.006528186927691823, "avg_duration_per_task_ms": 442659.0584149037, "avg_output_tokens_per_task": 29091.698137038205, "last_run_timestamp": "2026-10-31T16:02:14.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7698783076923077, "accuracy_std_dev": 0.010511058333485764, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.014805616938616937, "avg_duration_per_task_ms": 135269.72105672106, "avg_output_tokens_per_task": 29456.09829059829, "last_run_timestamp": "2026-10-31T16:51:23.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7694390259909032, "accuracy_std_dev": 0.01052704853876431, "run_count": 8, "total_questions": 1539, "avg_cost_per_task_usd": 0.02523249252761533, "avg_duration_per_task_ms": 74630.34632878493, "avg_output_tokens_per_task": 26425.37621832359, "last_run_timestamp": "2026-10-31T16:06:00.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7684847733333334, "accuracy_std_dev": 0.03945984809060131, "run_count": 9, "total_questions": 1650, "avg_cost_per_task_usd": 0.007370210909090909, "avg_duration_per_task_ms": 876615.1224242424, "avg_output_tokens_per_task": 18987.695151515152, "last_run_timestamp": "2026-10-31T16:04:29.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7669798106936416, "accuracy_std_dev": 0.011686708686296891, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.019905706647398845, "avg_duration_per_task_ms": 1154492.3439306358, "avg_output_tokens_per_task": 60181.387283236996, "last_run_timestamp": "2026-10-31T16:03:21.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7669488305084746, "accuracy_std_dev": 0.016365499999999977, "run_count": 2, "total_questions": 354, "avg_cost_per_task_usd": 0.21194037853107345, "avg_duration_per_task_ms": 159764.62146892655, "avg_output_tokens_per_task": 13982.706214689266, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.057140755050505046, "avg_duration_per_task_ms": 101024.92676767676, "avg_output_tokens_per_task": 22766.989898989897, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7653769053571428, "accuracy_std_dev": 0.011846747415441054, "run_count": 17, "total_questions": 3360, "avg_cost_per_task_usd": 0.016902593452380953, "avg_duration_per_task_ms": 110624.59017857142, "avg_output_tokens_per_task": 5988.1654761904765, "last_run_timestamp": "2026-10-31T09:06:38.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.76536225, "accuracy_std_dev": 0.00536920838555365, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004458252525252525, "avg_duration_per_task_ms": 487032.047979798, "avg_output_tokens_per_task": 29595.234848484848, "last_run_timestamp": "2026-10-31T16:16:55.698Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7651515, "accuracy_std_dev": 0.037753760451774886, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.052040077441077436, "avg_duration_per_task_ms": 879769.8745791246, "avg_output_tokens_per_task": 23186.810606060608, "last_run_timestamp": "2026-10-17T16:03:24.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.765058, "accuracy_std_dev": 0.01367922480747111, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.018581179573512904, "avg_duration_per_task_ms": 112140.46801346801, "avg_output_tokens_per_task": 30739.353535353537, "last_run_timestamp": "2026-10-31T16:52:10.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7649719016949154, "accuracy_std_dev": 0.008301844132480422, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.0026733169491525424, "avg_duration_per_task_ms": 102640.67457627118, "avg_output_tokens_per_task": 31256.29491525424, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "PrimeIntellect", "accuracy": 0.76431, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0026081363636363633, "avg_duration_per_task_ms": 302574.7474747475, "avg_output_tokens_per_task": 27415.863636363636, "last_run_timestamp": "2026-11-01T06:00:07.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Sail Research", "accuracy": 0.7640448089887641, "accuracy_std_dev": 0.007632810273199963, "run_count": 3, "total_questions": 534, "avg_cost_per_task_usd": 0.012456207865168538, "avg_duration_per_task_ms": 247061.07865168538, "avg_output_tokens_per_task": 31050.842696629214, "last_run_timestamp": "2026-11-06T21:57:37.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7638887500000001, "accuracy_std_dev": 0.01180708327392928, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02126011868686869, "avg_duration_per_task_ms": 1098413.3244949495, "avg_output_tokens_per_task": 53038.14141414141, "last_run_timestamp": "2026-10-31T16:51:21.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.7632998, "accuracy_std_dev": 0.007030464092789305, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007469111111111111, "avg_duration_per_task_ms": 123178.52121212121, "avg_output_tokens_per_task": 29681.70909090909, "last_run_timestamp": "2026-10-31T16:38:46.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7631873333333332, "accuracy_std_dev": 0.012472702469348371, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017184111111111112, "avg_duration_per_task_ms": 455850.7845117845, "avg_output_tokens_per_task": 17097.883838383837, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7629069999999999, "accuracy_std_dev": 0.00819966182229484, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2718667558922559, "avg_duration_per_task_ms": 264601.48148148146, "avg_output_tokens_per_task": 17934.003367003366, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7617845000000001, "accuracy_std_dev": 0.014110256482431497, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.01827727462121212, "avg_duration_per_task_ms": 146955.67613636365, "avg_output_tokens_per_task": 30220.935606060608, "last_run_timestamp": "2026-10-31T16:08:33.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7617845, "accuracy_std_dev": 0.011198548678288631, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017532584595959598, "avg_duration_per_task_ms": 239388.20833333334, "avg_output_tokens_per_task": 29069.847853535353, "last_run_timestamp": "2026-10-31T16:22:37.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7617845, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05473536363636364, "avg_duration_per_task_ms": 625909.4974747475, "avg_output_tokens_per_task": 22336.9797979798, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.761574, "accuracy_std_dev": 0.011842521785920426, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017385299873737374, "avg_duration_per_task_ms": 63084.142045454544, "avg_output_tokens_per_task": 28736.113636363636, "last_run_timestamp": "2026-10-31T16:23:59.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7614677541284404, "accuracy_std_dev": 0.10796943486314385, "run_count": 3, "total_questions": 545, "avg_cost_per_task_usd": 0.0038067559633027524, "avg_duration_per_task_ms": 346287.8605504587, "avg_output_tokens_per_task": 20923.759633027523, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7613168888888889, "accuracy_std_dev": 0.08989112326642029, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1361467480359147, "avg_duration_per_task_ms": 937300.8019079686, "avg_output_tokens_per_task": 71180.4657687991, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7607322499999999, "accuracy_std_dev": 0.04032017199166567, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.058705520202020196, "avg_duration_per_task_ms": 1045870.6470959596, "avg_output_tokens_per_task": 29725.0625, "last_run_timestamp": "2026-10-31T16:08:31.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7598906249999999, "accuracy_std_dev": 0.032074197830723274, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004828941287878788, "avg_duration_per_task_ms": 600231.7380050505, "avg_output_tokens_per_task": 28770.83648989899, "last_run_timestamp": "2026-10-31T16:20:12.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7592847364130435, "accuracy_std_dev": 0.03960627555022057, "run_count": 5, "total_questions": 736, "avg_cost_per_task_usd": 0.058288675271739136, "avg_duration_per_task_ms": 649092.0529891305, "avg_output_tokens_per_task": 49476.220108695656, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7574711434482759, "accuracy_std_dev": 0.05835625158448523, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.014400611954022986, "avg_duration_per_task_ms": 105760.68091954023, "avg_output_tokens_per_task": 6123.435402298851, "last_run_timestamp": "2026-10-31T08:19:36.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7570146666666666, "accuracy_std_dev": 0.017186872710169103, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05602859764309764, "avg_duration_per_task_ms": 578162.5252525252, "avg_output_tokens_per_task": 26055.28787878788, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7563452081218275, "accuracy_std_dev": 0.0358022938821275, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.014391308798646361, "avg_duration_per_task_ms": 1040556.6945854484, "avg_output_tokens_per_task": 51071.41201353638, "last_run_timestamp": "2026-10-31T08:41:02.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7558922500000002, "accuracy_std_dev": 0.007240774798838877, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022548712121212122, "avg_duration_per_task_ms": 94076.5808080808, "avg_output_tokens_per_task": 14873.041666666666, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.755892, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018171858585858586, "avg_duration_per_task_ms": 285760.0353535354, "avg_output_tokens_per_task": 24027.954545454544, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.755695906329114, "accuracy_std_dev": 0.006554500000000019, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.020412756962025318, "avg_duration_per_task_ms": 122185.9164556962, "avg_output_tokens_per_task": 11814.949367088608, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7547698333333334, "accuracy_std_dev": 0.0766049855914889, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004288400673400673, "avg_duration_per_task_ms": 447675.2971380471, "avg_output_tokens_per_task": 15291.021885521886, "last_run_timestamp": "2026-10-17T08:26:03.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7539042317380352, "accuracy_std_dev": 0.03122595132646972, "run_count": 12, "total_questions": 1985, "avg_cost_per_task_usd": 0.045778590428211585, "avg_duration_per_task_ms": 263292.8277078086, "avg_output_tokens_per_task": 17254.131486146096, "last_run_timestamp": "2026-10-31T08:39:08.315Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7525251818181818, "accuracy_std_dev": 0.013807926121801336, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.01597986134067952, "avg_duration_per_task_ms": 1293582.3039485766, "avg_output_tokens_per_task": 55438.24655647383, "last_run_timestamp": "2026-11-01T13:06:18.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7516448305921052, "accuracy_std_dev": 0.02929436769346626, "run_count": 5, "total_questions": 608, "avg_cost_per_task_usd": 0.051999368421052634, "avg_duration_per_task_ms": 386838.6842105263, "avg_output_tokens_per_task": 15199.644736842105, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7514029999999999, "accuracy_std_dev": 0.013279676389129373, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015289352693602695, "avg_duration_per_task_ms": 350945.7154882155, "avg_output_tokens_per_task": 62534.38636363636, "last_run_timestamp": "2026-10-31T16:38:47.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7502983046594983, "accuracy_std_dev": 0.030897500000000022, "run_count": 2, "total_questions": 279, "avg_cost_per_task_usd": 0.12269224014336919, "avg_duration_per_task_ms": 1216119.093189964, "avg_output_tokens_per_task": 34943.36559139785, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.749495, "accuracy_std_dev": 0.010721542221154592, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.013542626262626261, "avg_duration_per_task_ms": 93554.2, "avg_output_tokens_per_task": 14884.874747474747, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7494390818228898, "accuracy_std_dev": 0.10270027330445619, "run_count": 10, "total_questions": 1931, "avg_cost_per_task_usd": 0.006055046090108752, "avg_duration_per_task_ms": 549414.4293112376, "avg_output_tokens_per_task": 26366.88865872605, "last_run_timestamp": "2026-10-27T02:37:18.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7491585, "accuracy_std_dev": 0.009959720314848229, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014780738636363637, "avg_duration_per_task_ms": 104786.22727272728, "avg_output_tokens_per_task": 14546.537878787878, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.7481013645569621, "accuracy_std_dev": 0.004003499999999993, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.010130237974683544, "avg_duration_per_task_ms": 298779.6734177215, "avg_output_tokens_per_task": 60721.07341772152, "last_run_timestamp": "2026-10-10T08:39:12.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7455807499999999, "accuracy_std_dev": 0.025882017723460052, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1971899930555556, "avg_duration_per_task_ms": 1087151.1325757576, "avg_output_tokens_per_task": 89342.12184343435, "last_run_timestamp": "2026-10-31T16:01:53.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7450601862113403, "accuracy_std_dev": 0.02284683246884292, "run_count": 8, "total_questions": 1552, "avg_cost_per_task_usd": 0.016163423324742267, "avg_duration_per_task_ms": 377921.4826030928, "avg_output_tokens_per_task": 53593.9587628866, "last_run_timestamp": "2026-10-31T08:07:17.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7444443999999999, "accuracy_std_dev": 0.008741194188439036, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006576761616161616, "avg_duration_per_task_ms": 163397.33737373736, "avg_output_tokens_per_task": 7642.029292929293, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7441077777777777, "accuracy_std_dev": 0.02634512333864457, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.33108891245791244, "avg_duration_per_task_ms": 924653.0869809203, "avg_output_tokens_per_task": 82641.20707070707, "last_run_timestamp": "2026-10-31T16:39:49.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7421462352941177, "accuracy_std_dev": 0.08394561417332949, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.004914519924098672, "avg_duration_per_task_ms": 596355.0860215054, "avg_output_tokens_per_task": 32270, "last_run_timestamp": "2026-10-17T16:03:21.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7421436666666666, "accuracy_std_dev": 0.0029824300867283098, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0052554671717171715, "avg_duration_per_task_ms": 138780.79882154881, "avg_output_tokens_per_task": 8488.298821548822, "last_run_timestamp": "2026-10-17T16:06:15.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.741847766304348, "accuracy_std_dev": 0.004303026282455164, "run_count": 4, "total_questions": 736, "avg_cost_per_task_usd": 0.008672221467391304, "avg_duration_per_task_ms": 124088.47690217392, "avg_output_tokens_per_task": 10910.259510869566, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7401770526570047, "accuracy_std_dev": 0.0865423410071625, "run_count": 12, "total_questions": 2070, "avg_cost_per_task_usd": 0.0039563893719806766, "avg_duration_per_task_ms": 552542.9028985507, "avg_output_tokens_per_task": 24944.29468599034, "last_run_timestamp": "2026-11-04T18:42:44.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7398992499999999, "accuracy_std_dev": 0.0060112973797259595, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02330285101010101, "avg_duration_per_task_ms": 95915.94318181818, "avg_output_tokens_per_task": 15426.907828282829, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.739899, "accuracy_std_dev": 0.007575777781851822, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0048982929292929285, "avg_duration_per_task_ms": 108030.26641414141, "avg_output_tokens_per_task": 7988.015151515152, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7385018107983389, "accuracy_std_dev": 0.1187447930907569, "run_count": 12, "total_questions": 2167, "avg_cost_per_task_usd": 0.033791137517305024, "avg_duration_per_task_ms": 1331765.8463313337, "avg_output_tokens_per_task": 35484.031379787724, "last_run_timestamp": "2026-10-31T16:04:48.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.7384959999999999, "accuracy_std_dev": 0.008170584087648423, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06071549915824916, "avg_duration_per_task_ms": 125902.70959595959, "avg_output_tokens_per_task": 24199.07239057239, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.738005, "accuracy_std_dev": 0.009886114694357948, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.11405846527777778, "avg_duration_per_task_ms": 757664.5296717172, "avg_output_tokens_per_task": 44516.394570707074, "last_run_timestamp": "2026-10-31T16:04:48.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7368489013657057, "accuracy_std_dev": 0.05193843550644497, "run_count": 7, "total_questions": 1318, "avg_cost_per_task_usd": 0.018259321699544768, "avg_duration_per_task_ms": 840685.5872534142, "avg_output_tokens_per_task": 45854.41881638847, "last_run_timestamp": "2026-10-31T16:47:37.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7366410572519084, "accuracy_std_dev": 0.012366834942068835, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.006647374045801527, "avg_duration_per_task_ms": 328667.01463104325, "avg_output_tokens_per_task": 26397.287531806614, "last_run_timestamp": "2026-10-31T16:16:13.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.736532, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07812436868686869, "avg_duration_per_task_ms": 803892.6035353536, "avg_output_tokens_per_task": 22434.27525252525, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7363656367656349, "accuracy_std_dev": 0.008751528723600225, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.007021190145293747, "avg_duration_per_task_ms": 251648.8812381554, "avg_output_tokens_per_task": 8683.42135186355, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.735900625, "accuracy_std_dev": 0.023012779036534772, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006714852904040405, "avg_duration_per_task_ms": 396625.5202020202, "avg_output_tokens_per_task": 22776.6452020202, "last_run_timestamp": "2026-10-31T16:08:31.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7328042857142857, "accuracy_std_dev": 0.006328935223339255, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006012205627705627, "avg_duration_per_task_ms": 432462.59451659454, "avg_output_tokens_per_task": 7938.487734487734, "last_run_timestamp": "2026-10-31T16:08:56.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.732440575, "accuracy_std_dev": 0.057246596407316816, "run_count": 3, "total_questions": 560, "avg_cost_per_task_usd": 0.014524144642857144, "avg_duration_per_task_ms": 428029.5660714286, "avg_output_tokens_per_task": 58878.205357142855, "last_run_timestamp": "2026-10-31T08:03:10.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.732155, "accuracy_std_dev": 0.02424727451075689, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.016461675757575758, "avg_duration_per_task_ms": 673272.2666666667, "avg_output_tokens_per_task": 60883.92525252525, "last_run_timestamp": "2026-10-31T08:31:11.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7306920882647944, "accuracy_std_dev": 0.018533662569629808, "run_count": 7, "total_questions": 997, "avg_cost_per_task_usd": 0.016062558676028082, "avg_duration_per_task_ms": 67044.65897693079, "avg_output_tokens_per_task": 9235.0962888666, "last_run_timestamp": "2026-10-31T16:30:53.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7305295887850467, "accuracy_std_dev": 0.01062550000000001, "run_count": 2, "total_questions": 321, "avg_cost_per_task_usd": 0.08696355451713396, "avg_duration_per_task_ms": 1922851.7601246105, "avg_output_tokens_per_task": 28235.09968847352, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7304292499999999, "accuracy_std_dev": 0.021067858379234947, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3208809797979798, "avg_duration_per_task_ms": 1030227.9185606061, "avg_output_tokens_per_task": 94983.5441919192, "last_run_timestamp": "2026-10-31T16:11:15.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7303591666666667, "accuracy_std_dev": 0.00844092435531926, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1903736430976431, "avg_duration_per_task_ms": 238390.79924242425, "avg_output_tokens_per_task": 37883.638468013465, "last_run_timestamp": "2026-10-31T08:20:09.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7303417538461537, "accuracy_std_dev": 0.11666421720821496, "run_count": 6, "total_questions": 1170, "avg_cost_per_task_usd": 0.03974357521367521, "avg_duration_per_task_ms": 502126.6564102564, "avg_output_tokens_per_task": 35593.947863247864, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7301687421940927, "accuracy_std_dev": 0.019520676718688693, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.0065871569620253164, "avg_duration_per_task_ms": 133816.87552742616, "avg_output_tokens_per_task": 6899.4869198312235, "last_run_timestamp": "2026-10-17T16:03:44.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7299572432432432, "accuracy_std_dev": 0.012015924167099304, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.20459208230958226, "avg_duration_per_task_ms": 238486.2523887524, "avg_output_tokens_per_task": 38144.53794703795, "last_run_timestamp": "2026-10-31T08:33:11.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7281145, "accuracy_std_dev": 0.004208499999999948, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.060785368686868686, "avg_duration_per_task_ms": 121296.46212121213, "avg_output_tokens_per_task": 24226.65404040404, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7271808509532062, "accuracy_std_dev": 0.03540222199908109, "run_count": 6, "total_questions": 1154, "avg_cost_per_task_usd": 0.011152869150779895, "avg_duration_per_task_ms": 504501.5831889082, "avg_output_tokens_per_task": 55506.62998266898, "last_run_timestamp": "2026-10-10T08:24:27.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7268518333333334, "accuracy_std_dev": 0.0122393019874047, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.18790157449494946, "avg_duration_per_task_ms": 344629.31313131313, "avg_output_tokens_per_task": 37389.22474747475, "last_run_timestamp": "2026-10-31T08:16:11.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.7267116666666668, "accuracy_std_dev": 0.01049864519301841, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.021460314814814814, "avg_duration_per_task_ms": 244130.04882154881, "avg_output_tokens_per_task": 76264.67003367003, "last_run_timestamp": "2026-10-31T08:19:36.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7260101666666667, "accuracy_std_dev": 0.010493774144965926, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.18633288762626263, "avg_duration_per_task_ms": 306780.9974747475, "avg_output_tokens_per_task": 37097.41203703704, "last_run_timestamp": "2026-10-31T08:09:30.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.725589375, "accuracy_std_dev": 0.015859785220310358, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005495951388888889, "avg_duration_per_task_ms": 318269.0252525252, "avg_output_tokens_per_task": 8055.998106060606, "last_run_timestamp": "2026-10-31T16:22:55.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7249578750000001, "accuracy_std_dev": 0.041655408686740475, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3217009703282828, "avg_duration_per_task_ms": 1291699.7613636365, "avg_output_tokens_per_task": 101576.43434343435, "last_run_timestamp": "2026-10-31T16:18:48.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.7236893527788727, "accuracy_std_dev": 0.03777209883376474, "run_count": 13, "total_questions": 2537, "avg_cost_per_task_usd": 0.010590881750098544, "avg_duration_per_task_ms": 263440.8253843122, "avg_output_tokens_per_task": 48345.98265668112, "last_run_timestamp": "2026-10-31T08:46:43.680Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.723310259001895, "accuracy_std_dev": 0.03972893833844543, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.31632930574857865, "avg_duration_per_task_ms": 1267532.5773847126, "avg_output_tokens_per_task": 99778.44725205307, "last_run_timestamp": "2026-10-31T16:18:28.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.722643, "accuracy_std_dev": 0.0177518271735616, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004502867424242425, "avg_duration_per_task_ms": 166323.601010101, "avg_output_tokens_per_task": 7638.002525252526, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7221286666666666, "accuracy_std_dev": 0.010833865812554817, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00440245847362514, "avg_duration_per_task_ms": 697842.308080808, "avg_output_tokens_per_task": 7879.701459034793, "last_run_timestamp": "2026-10-31T16:02:18.363Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7218013888888889, "accuracy_std_dev": 0.013975083045425636, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.19767881116722782, "avg_duration_per_task_ms": 272001.3790684624, "avg_output_tokens_per_task": 36868.61924803592, "last_run_timestamp": "2026-10-31T08:29:35.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7210140000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 92, "avg_cost_per_task_usd": 0.08365161956521738, "avg_duration_per_task_ms": 673616.0108695652, "avg_output_tokens_per_task": 34073.82608695652, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7207904490979381, "accuracy_std_dev": 0.05082374714542744, "run_count": 16, "total_questions": 3104, "avg_cost_per_task_usd": 0.008789419136597938, "avg_duration_per_task_ms": 550071.2590206185, "avg_output_tokens_per_task": 26091.442332474227, "last_run_timestamp": "2026-10-31T16:52:55.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7183742857142856, "accuracy_std_dev": 0.012394494661678109, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01898589466089466, "avg_duration_per_task_ms": 501749.6746031746, "avg_output_tokens_per_task": 67465.82683982684, "last_run_timestamp": "2026-10-31T08:06:04.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7180135, "accuracy_std_dev": 0.002525166677667045, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.011038688131313132, "avg_duration_per_task_ms": 91885.46717171717, "avg_output_tokens_per_task": 10364.281565656565, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7176046841432225, "accuracy_std_dev": 0.014357802800219841, "run_count": 4, "total_questions": 782, "avg_cost_per_task_usd": 0.011155867007672635, "avg_duration_per_task_ms": 100550.45524296675, "avg_output_tokens_per_task": 10241.003836317135, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.715488, "accuracy_std_dev": 0.008584199351133475, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007857320707070707, "avg_duration_per_task_ms": 80203.69570707071, "avg_output_tokens_per_task": 9977.103535353535, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7112795000000001, "accuracy_std_dev": 0.008459507683665782, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008834092171717172, "avg_duration_per_task_ms": 129721.28282828283, "avg_output_tokens_per_task": 5726.907828282829, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7104377500000001, "accuracy_std_dev": 0.0035712428071331226, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015488117424242423, "avg_duration_per_task_ms": 97440.21590909091, "avg_output_tokens_per_task": 10244.169191919193, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.71001675, "accuracy_std_dev": 0.018948683824675006, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015486457070707072, "avg_duration_per_task_ms": 1105788.4797979798, "avg_output_tokens_per_task": 64321.568181818184, "last_run_timestamp": "2026-10-31T16:02:14.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7074915000000002, "accuracy_std_dev": 0.007657142629597565, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0059589507575757575, "avg_duration_per_task_ms": 108794.86363636363, "avg_output_tokens_per_task": 5529.719696969697, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.7070704999999999, "accuracy_std_dev": 0.025252499999999956, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06934866919191919, "avg_duration_per_task_ms": 1041055.3939393939, "avg_output_tokens_per_task": 23164.866161616163, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7059485833333332, "accuracy_std_dev": 0.13670659769524554, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.615879143097643, "avg_duration_per_task_ms": 638649.8851010101, "avg_output_tokens_per_task": 40901.45538720539, "last_run_timestamp": "2026-10-31T08:09:30.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7051066666666668, "accuracy_std_dev": 0.14363924551362076, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.619232220959596, "avg_duration_per_task_ms": 639502.4095117846, "avg_output_tokens_per_task": 41183.20328282828, "last_run_timestamp": "2026-10-31T08:28:17.671Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7049665, "accuracy_std_dev": 0.017307340935279462, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.010932695707070707, "avg_duration_per_task_ms": 188922.1123737374, "avg_output_tokens_per_task": 5690.835858585859, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.704655008, "accuracy_std_dev": 0.13819566036718844, "run_count": 36, "total_questions": 7125, "avg_cost_per_task_usd": 0.6637001717894737, "avg_duration_per_task_ms": 604263.9247719299, "avg_output_tokens_per_task": 41325.50835087719, "last_run_timestamp": "2026-10-31T08:51:32.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.7038606610169492, "accuracy_std_dev": 0.025828434379814764, "run_count": 11, "total_questions": 2124, "avg_cost_per_task_usd": 0.009122728342749528, "avg_duration_per_task_ms": 365582.73399246705, "avg_output_tokens_per_task": 45350.33474576271, "last_run_timestamp": "2026-10-31T08:03:09.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.703844, "accuracy_std_dev": 0.13064484273849722, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.611304741161616, "avg_duration_per_task_ms": 637806.0361952862, "avg_output_tokens_per_task": 40663.66961279461, "last_run_timestamp": "2026-10-31T08:38:33.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7025463750000001, "accuracy_std_dev": 0.0578804880549946, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13540828724747475, "avg_duration_per_task_ms": 664851.7664141414, "avg_output_tokens_per_task": 40688.72285353536, "last_run_timestamp": "2026-10-31T16:56:31.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.7024950168823861, "accuracy_std_dev": 0.09479570165959954, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.011357701181767021, "avg_duration_per_task_ms": 821641.8733821047, "avg_output_tokens_per_task": 33243.20652785594, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7024297297297296, "accuracy_std_dev": 0.13362786647407585, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.6591401015561016, "avg_duration_per_task_ms": 440446.8645918646, "avg_output_tokens_per_task": 40987.61315861316, "last_run_timestamp": "2026-10-31T08:31:32.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.701915125, "accuracy_std_dev": 0.04741224100492801, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.015016424242424244, "avg_duration_per_task_ms": 909166.5378787878, "avg_output_tokens_per_task": 50028.55808080808, "last_run_timestamp": "2026-10-31T16:11:46.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7003709888776543, "accuracy_std_dev": 0.00943615013869532, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.010341509605662285, "avg_duration_per_task_ms": 140930.19615773507, "avg_output_tokens_per_task": 9335.02628918099, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.6998505497757848, "accuracy_std_dev": 0.1222266014861715, "run_count": 8, "total_questions": 1115, "avg_cost_per_task_usd": 0.09317169596412556, "avg_duration_per_task_ms": 1222512.3695067265, "avg_output_tokens_per_task": 26526.149775784754, "last_run_timestamp": "2026-10-31T16:05:14.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.6973907500000001, "accuracy_std_dev": 0.0320390349837117, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01086133207070707, "avg_duration_per_task_ms": 63422.920454545456, "avg_output_tokens_per_task": 9015.968434343435, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6970787019475021, "accuracy_std_dev": 0.046119854611155975, "run_count": 12, "total_questions": 2362, "avg_cost_per_task_usd": 0.007317635478408128, "avg_duration_per_task_ms": 292695.22861981374, "avg_output_tokens_per_task": 40004.958086367486, "last_run_timestamp": "2026-10-31T08:15:24.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.6947253333333333, "accuracy_std_dev": 0.06031432281712934, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0065146380471380475, "avg_duration_per_task_ms": 125260.52525252526, "avg_output_tokens_per_task": 7227.368686868687, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6922558, "accuracy_std_dev": 0.010774381251839934, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00555330606060606, "avg_duration_per_task_ms": 393393.0888888889, "avg_output_tokens_per_task": 6125.930303030303, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.6907968888888889, "accuracy_std_dev": 0.011334983541079488, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006490662738496071, "avg_duration_per_task_ms": 110618.54152637486, "avg_output_tokens_per_task": 15950.401234567902, "last_run_timestamp": "2026-10-31T16:08:56.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6885522307692307, "accuracy_std_dev": 0.1455903473845781, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.6175294277389277, "avg_duration_per_task_ms": 635458.5120435121, "avg_output_tokens_per_task": 41042.69852369852, "last_run_timestamp": "2026-10-31T08:22:47.508Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6876397825763216, "accuracy_std_dev": 0.02033882284635741, "run_count": 7, "total_questions": 1343, "avg_cost_per_task_usd": 0.01665832092330603, "avg_duration_per_task_ms": 832849.0714817572, "avg_output_tokens_per_task": 83099.43037974683, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.6866834096080909, "accuracy_std_dev": 0.0076575271261354015, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.007103366624525917, "avg_duration_per_task_ms": 1621893.5347661187, "avg_output_tokens_per_task": 50543.86852085967, "last_run_timestamp": "2026-10-31T16:36:31.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6858604107485604, "accuracy_std_dev": 0.04695142144480293, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.02905995585412668, "avg_duration_per_task_ms": 808566.8777991043, "avg_output_tokens_per_task": 18265.64043506078, "last_run_timestamp": "2026-10-31T16:23:19.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.6843435, "accuracy_std_dev": 0.01913932645758466, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.009632915404040404, "avg_duration_per_task_ms": 272984.89015151514, "avg_output_tokens_per_task": 6630.618686868687, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6821764194977844, "accuracy_std_dev": 0.028397019030270698, "run_count": 6, "total_questions": 677, "avg_cost_per_task_usd": 0.008931531757754801, "avg_duration_per_task_ms": 55086.47267355982, "avg_output_tokens_per_task": 5789.069423929099, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.6816816949806949, "accuracy_std_dev": 0.019977202224895615, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.002334259330759331, "avg_duration_per_task_ms": 214658.21685971686, "avg_output_tokens_per_task": 4092.794723294723, "last_run_timestamp": "2026-10-31T16:18:48.428Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.681502625, "accuracy_std_dev": 0.06815065878246794, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007994105429292929, "avg_duration_per_task_ms": 1440482.744949495, "avg_output_tokens_per_task": 62597.993055555555, "last_run_timestamp": "2026-10-31T16:08:08.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6785352626262626, "accuracy_std_dev": 0.014156861524116758, "run_count": 13, "total_questions": 1980, "avg_cost_per_task_usd": 0.005069473232323233, "avg_duration_per_task_ms": 756595.5121212121, "avg_output_tokens_per_task": 50528.773232323234, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.677734640625, "accuracy_std_dev": 0.0033374214597500216, "run_count": 3, "total_questions": 512, "avg_cost_per_task_usd": 0.04002998046875, "avg_duration_per_task_ms": 677472.7421875, "avg_output_tokens_per_task": 42379.095703125, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6770081428571428, "accuracy_std_dev": 0.13047305606076964, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007556641414141414, "avg_duration_per_task_ms": 737595.6847041846, "avg_output_tokens_per_task": 26027.38023088023, "last_run_timestamp": "2026-10-31T16:45:39.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6764644057507988, "accuracy_std_dev": 0.06386519178597505, "run_count": 8, "total_questions": 1565, "avg_cost_per_task_usd": 0.05298015335463258, "avg_duration_per_task_ms": 1168436.861341853, "avg_output_tokens_per_task": 24293.87412140575, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6691934983520105, "accuracy_std_dev": 0.018592824882494627, "run_count": 9, "total_questions": 1517, "avg_cost_per_task_usd": 0.00669224719841793, "avg_duration_per_task_ms": 471626.4996704021, "avg_output_tokens_per_task": 47630.16677653263, "last_run_timestamp": "2026-10-31T16:03:21.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.667718875, "accuracy_std_dev": 0.01755485164731891, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007828616792929293, "avg_duration_per_task_ms": 514528.57007575757, "avg_output_tokens_per_task": 60437.62941919192, "last_run_timestamp": "2026-10-31T16:08:31.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6677188749999998, "accuracy_std_dev": 0.006886930456260996, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00510659595959596, "avg_duration_per_task_ms": 104136.78345959596, "avg_output_tokens_per_task": 9617.265151515152, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.66726417562724, "accuracy_std_dev": 0.001755225531567576, "run_count": 3, "total_questions": 279, "avg_cost_per_task_usd": 0.004358928315412186, "avg_duration_per_task_ms": 566517.577060932, "avg_output_tokens_per_task": 29219.63799283154, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.667228, "accuracy_std_dev": 0.006780740569190578, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0024873282828282825, "avg_duration_per_task_ms": 87761.60606060606, "avg_output_tokens_per_task": 2304.691919191919, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6672276666666667, "accuracy_std_dev": 0.006918527315998866, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002891920875420875, "avg_duration_per_task_ms": 83806.60269360269, "avg_output_tokens_per_task": 3112.5656565656564, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6670875, "accuracy_std_dev": 0.016207809136339183, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005778919191919192, "avg_duration_per_task_ms": 411871.7727272727, "avg_output_tokens_per_task": 15840.212121212122, "last_run_timestamp": "2026-10-31T16:17:15.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6666666666666666, "accuracy_std_dev": 0.020248820744812664, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005717542087542088, "avg_duration_per_task_ms": 446868.9478114478, "avg_output_tokens_per_task": 16225.250841750842, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.666246, "accuracy_std_dev": 0.052274901764613566, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01620974368686869, "avg_duration_per_task_ms": 334155.6628787879, "avg_output_tokens_per_task": 8982.006313131313, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.665825, "accuracy_std_dev": 0.0075755555571852436, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.002092334595959596, "avg_duration_per_task_ms": 237098.96464646465, "avg_output_tokens_per_task": 3099.743686868687, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6643520625000001, "accuracy_std_dev": 0.013584297200576622, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.009127087752525253, "avg_duration_per_task_ms": 320481.77809343435, "avg_output_tokens_per_task": 60446.78566919192, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6626460482456141, "accuracy_std_dev": 0.07409874774620387, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.0063980043859649125, "avg_duration_per_task_ms": 453491.5789473684, "avg_output_tokens_per_task": 34112.30263157895, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6603534999999999, "accuracy_std_dev": 0.021669912920914115, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008152684974747474, "avg_duration_per_task_ms": 795790.2424242424, "avg_output_tokens_per_task": 55529.520833333336, "last_run_timestamp": "2026-10-31T16:08:56.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6599326666666666, "accuracy_std_dev": 0.013255807590972711, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015927558922558923, "avg_duration_per_task_ms": 14095.8367003367, "avg_output_tokens_per_task": 1794.7289562289561, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.657360497461929, "accuracy_std_dev": 0.009259500000000032, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.04522630456852792, "avg_duration_per_task_ms": 3082748.9060913706, "avg_output_tokens_per_task": 28446.494923857867, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.654250875, "accuracy_std_dev": 0.13793927785482052, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.014789621212121212, "avg_duration_per_task_ms": 422898.68813131313, "avg_output_tokens_per_task": 12693.433080808081, "last_run_timestamp": "2026-10-31T16:01:53.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6531986666666667, "accuracy_std_dev": 0.009013858120817182, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0030076632996633, "avg_duration_per_task_ms": 41226.700336700334, "avg_output_tokens_per_task": 2842.754208754209, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.652357, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004161952020202021, "avg_duration_per_task_ms": 27160.11111111111, "avg_output_tokens_per_task": 2408.8232323232323, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.651515, "accuracy_std_dev": 0.1905456270048375, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.058297676767676776, "avg_duration_per_task_ms": 1098090.5791245792, "avg_output_tokens_per_task": 23174.20202020202, "last_run_timestamp": "2026-10-17T16:03:44.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6508838749999999, "accuracy_std_dev": 0.023523002829345063, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005865840909090909, "avg_duration_per_task_ms": 657820.1502525252, "avg_output_tokens_per_task": 42824.80997474748, "last_run_timestamp": "2026-10-31T16:01:53.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.010498645193018418, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0041127255892255896, "avg_duration_per_task_ms": 24949.132996632998, "avg_output_tokens_per_task": 2375.9343434343436, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.6484773807106599, "accuracy_std_dev": 0.0054344006339245585, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.00290907614213198, "avg_duration_per_task_ms": 58072.35659898477, "avg_output_tokens_per_task": 2371.812182741117, "last_run_timestamp": "2026-10-27T00:30:02.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.6479142515802782, "accuracy_std_dev": 0.10866298734612617, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.009831766750948167, "avg_duration_per_task_ms": 374447.82616940583, "avg_output_tokens_per_task": 27636.46776232617, "last_run_timestamp": "2026-10-31T16:12:07.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.6478676666666666, "accuracy_std_dev": 0.008199376656524262, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0038777457912457913, "avg_duration_per_task_ms": 63026.43939393939, "avg_output_tokens_per_task": 2350.077441077441, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6477275, "accuracy_std_dev": 0.005503645223849386, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0027929646464646468, "avg_duration_per_task_ms": 157689.33585858587, "avg_output_tokens_per_task": 9994.479797979799, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6467880041128085, "accuracy_std_dev": 0.20710201321517321, "run_count": 9, "total_questions": 1702, "avg_cost_per_task_usd": 0.011494767920094007, "avg_duration_per_task_ms": 300785.6809635723, "avg_output_tokens_per_task": 13387.064629847238, "last_run_timestamp": "2026-10-17T08:17:57.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6453423333333334, "accuracy_std_dev": 0.010677151752951496, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0023666683501683498, "avg_duration_per_task_ms": 54700.6936026936, "avg_output_tokens_per_task": 2768.026936026936, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.6436586666666667, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002485070707070707, "avg_duration_per_task_ms": 52022.1632996633, "avg_output_tokens_per_task": 2681.813131313131, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.64025675, "accuracy_std_dev": 0.01324879906585875, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.001088048611111111, "avg_duration_per_task_ms": 331146.75505050505, "avg_output_tokens_per_task": 11572.473484848484, "last_run_timestamp": "2026-10-31T16:31:13.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.640151625, "accuracy_std_dev": 0.013825012268868864, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007202421717171718, "avg_duration_per_task_ms": 241781.60606060605, "avg_output_tokens_per_task": 11537.880681818182, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6401515, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004124544191919192, "avg_duration_per_task_ms": 34476.35353535353, "avg_output_tokens_per_task": 2259.632575757576, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.63846825, "accuracy_std_dev": 0.015215861565731332, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0046192222222222225, "avg_duration_per_task_ms": 77763.38257575757, "avg_output_tokens_per_task": 8686.127525252525, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6372055000000001, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015519383838383837, "avg_duration_per_task_ms": 14506.310606060606, "avg_output_tokens_per_task": 1746.2525252525252, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.6367847499999999, "accuracy_std_dev": 0.012277609526593496, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015237281565656567, "avg_duration_per_task_ms": 18840.242424242424, "avg_output_tokens_per_task": 1618.909090909091, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6361454238683127, "accuracy_std_dev": 0.218773277941707, "run_count": 8, "total_questions": 1458, "avg_cost_per_task_usd": 0.00909512757201646, "avg_duration_per_task_ms": 157677.57133058985, "avg_output_tokens_per_task": 14981.154320987655, "last_run_timestamp": "2026-10-31T16:54:17.849Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6351663037907099, "accuracy_std_dev": 0.019472921033065377, "run_count": 10, "total_questions": 1873, "avg_cost_per_task_usd": 0.005367699412706888, "avg_duration_per_task_ms": 807188.147357181, "avg_output_tokens_per_task": 35119.06513614522, "last_run_timestamp": "2026-10-31T08:20:09.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6350172000000001, "accuracy_std_dev": 0.013898819394466565, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0016177010101010101, "avg_duration_per_task_ms": 77808.62626262626, "avg_output_tokens_per_task": 8177.061616161616, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.63320725, "accuracy_std_dev": 0.035387886365781995, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006817837121212121, "avg_duration_per_task_ms": 104450.63320707071, "avg_output_tokens_per_task": 9507.679924242424, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6328369143389199, "accuracy_std_dev": 0.012371127066233214, "run_count": 3, "total_questions": 537, "avg_cost_per_task_usd": 0.00797138175046555, "avg_duration_per_task_ms": 28551.49720670391, "avg_output_tokens_per_task": 13203.731843575419, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6314185, "accuracy_std_dev": 0.01109086049411856, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.019078623737373738, "avg_duration_per_task_ms": 506361.1357323232, "avg_output_tokens_per_task": 95407.51325757576, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6306148156028369, "accuracy_std_dev": 0.05281625698168657, "run_count": 3, "total_questions": 282, "avg_cost_per_task_usd": 0.0037584255319148938, "avg_duration_per_task_ms": 208452.59574468085, "avg_output_tokens_per_task": 12308.468085106382, "last_run_timestamp": "2026-10-23T09:29:14.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6305678657378088, "accuracy_std_dev": 0.007960405916746635, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.0018545345155161492, "avg_duration_per_task_ms": 230896.0607979734, "avg_output_tokens_per_task": 10115.67764407853, "last_run_timestamp": "2026-10-31T16:10:09.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.62962975, "accuracy_std_dev": 0.007985647073813095, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0027725871212121215, "avg_duration_per_task_ms": 281279.59217171714, "avg_output_tokens_per_task": 9027.892676767677, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.6295420719476744, "accuracy_std_dev": 0.1792338661514529, "run_count": 14, "total_questions": 2752, "avg_cost_per_task_usd": 0.02676043095930232, "avg_duration_per_task_ms": 725458.0534156977, "avg_output_tokens_per_task": 37903.851744186046, "last_run_timestamp": "2026-10-31T16:08:33.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.629208875, "accuracy_std_dev": 0.022106924353228682, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017882390151515153, "avg_duration_per_task_ms": 634871.8630050505, "avg_output_tokens_per_task": 87147.22348484848, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.627104375, "accuracy_std_dev": 0.008333089267155095, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024497946969696968, "avg_duration_per_task_ms": 1060409.1414141415, "avg_output_tokens_per_task": 101874.34027777778, "last_run_timestamp": "2026-10-23T09:03:34.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6240183333333333, "accuracy_std_dev": 0.009654676391377534, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0036675572390572394, "avg_duration_per_task_ms": 146319.07912457912, "avg_output_tokens_per_task": 12576.434343434343, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6236870803858521, "accuracy_std_dev": 0.010127786036444495, "run_count": 8, "total_questions": 1555, "avg_cost_per_task_usd": 0.016994219935691317, "avg_duration_per_task_ms": 65807.6733118971, "avg_output_tokens_per_task": 56415.3536977492, "last_run_timestamp": "2026-10-31T16:52:47.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6232324, "accuracy_std_dev": 0.01591442443319895, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004449116161616162, "avg_duration_per_task_ms": 258518.11111111112, "avg_output_tokens_per_task": 14559.569696969696, "last_run_timestamp": "2026-10-23T08:08:11.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6182949425287356, "accuracy_std_dev": 0.0013475000000000015, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.011105244252873564, "avg_duration_per_task_ms": 345775.4454022989, "avg_output_tokens_per_task": 38606.68103448276, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6150393333333333, "accuracy_std_dev": 0.00649574344389378, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005692112794612795, "avg_duration_per_task_ms": 660625.7861952862, "avg_output_tokens_per_task": 18940.488215488214, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6134706293436293, "accuracy_std_dev": 0.013891280772484577, "run_count": 4, "total_questions": 777, "avg_cost_per_task_usd": 0.01592537323037323, "avg_duration_per_task_ms": 820331.4504504504, "avg_output_tokens_per_task": 16891.465894465895, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6123375670628184, "accuracy_std_dev": 0.008273645159312946, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.003321100169779287, "avg_duration_per_task_ms": 426798.44821731746, "avg_output_tokens_per_task": 13430.818336162989, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6121632499999999, "accuracy_std_dev": 0.004096520863794059, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004063032828282829, "avg_duration_per_task_ms": 22356.58080808081, "avg_output_tokens_per_task": 2158.7285353535353, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00778269696969697, "avg_duration_per_task_ms": 302947.1666666667, "avg_output_tokens_per_task": 15369.68686868687, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6077442500000001, "accuracy_std_dev": 0.01208155948905189, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.003638679292929293, "avg_duration_per_task_ms": 32468.945707070707, "avg_output_tokens_per_task": 2148.3560606060605, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6069025, "accuracy_std_dev": 0.006127792118699825, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0035186742424242424, "avg_duration_per_task_ms": 45117.36994949495, "avg_output_tokens_per_task": 2127.872474747475, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Mara", "accuracy": 0.6059481003717473, "accuracy_std_dev": 0.024067533945591992, "run_count": 3, "total_questions": 269, "avg_cost_per_task_usd": 0.03276318587360595, "avg_duration_per_task_ms": 68520.27137546468, "avg_output_tokens_per_task": 7032.501858736059, "last_run_timestamp": "2026-10-31T16:08:33.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6043773333333333, "accuracy_std_dev": 0.010910481270574418, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008336993265993266, "avg_duration_per_task_ms": 317962.5740740741, "avg_output_tokens_per_task": 17357.656565656565, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6037849337670017, "accuracy_std_dev": 0.011765991415941101, "run_count": 9, "total_questions": 1691, "avg_cost_per_task_usd": 0.008547784742755766, "avg_duration_per_task_ms": 560925.4151389711, "avg_output_tokens_per_task": 14044.128326434062, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6026385750873108, "accuracy_std_dev": 0.021655756090240754, "run_count": 5, "total_questions": 859, "avg_cost_per_task_usd": 0.0021882596041909196, "avg_duration_per_task_ms": 53677.2398137369, "avg_output_tokens_per_task": 2050.151338766007, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6010102499999999, "accuracy_std_dev": 0.0035714785016152262, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.011003395202020202, "avg_duration_per_task_ms": 31387.02398989899, "avg_output_tokens_per_task": 2097.1805555555557, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.6010099999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04278589646464647, "avg_duration_per_task_ms": 337432.404040404, "avg_output_tokens_per_task": 21288.575757575756, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.600457, "accuracy_std_dev": null, "run_count": 1, "total_questions": 146, "avg_cost_per_task_usd": 0.06895466438356164, "avg_duration_per_task_ms": 858101.3082191781, "avg_output_tokens_per_task": 24023.73287671233, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.599327, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014991068181818182, "avg_duration_per_task_ms": 284433.01515151514, "avg_output_tokens_per_task": 12302.075757575758, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5978837142857143, "accuracy_std_dev": 0.01426892240114143, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0030797525252525253, "avg_duration_per_task_ms": 643402.1818181818, "avg_output_tokens_per_task": 13628.22077922078, "last_run_timestamp": "2026-11-01T11:08:51.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.5972300452232899, "accuracy_std_dev": 0.06739364692023929, "run_count": 9, "total_questions": 1769, "avg_cost_per_task_usd": 0.012534433578292821, "avg_duration_per_task_ms": 1310890.6302996043, "avg_output_tokens_per_task": 57642.5850763143, "last_run_timestamp": "2026-10-31T16:05:14.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.5972225000000001, "accuracy_std_dev": 0.00323265575804168, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0032439330808080808, "avg_duration_per_task_ms": 69237.93308080808, "avg_output_tokens_per_task": 2070.3876262626263, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5963805, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01735105808080808, "avg_duration_per_task_ms": 112232.29292929293, "avg_output_tokens_per_task": 43198.833333333336, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5955725338441891, "accuracy_std_dev": 0.008950333597134817, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.009035680076628352, "avg_duration_per_task_ms": 471811.50638569606, "avg_output_tokens_per_task": 44984.78607918263, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5892258750000001, "accuracy_std_dev": 0.008204338493100791, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01031350505050505, "avg_duration_per_task_ms": 10608.848484848484, "avg_output_tokens_per_task": 13296.869318181818, "last_run_timestamp": "2026-10-31T16:21:16.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5873616193353474, "accuracy_std_dev": 0.048784153240032144, "run_count": 7, "total_questions": 662, "avg_cost_per_task_usd": 0.03112247280966767, "avg_duration_per_task_ms": 62861.41993957704, "avg_output_tokens_per_task": 6675.285498489426, "last_run_timestamp": "2026-10-31T16:41:30.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.58585875, "accuracy_std_dev": 0.007241239789393828, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006404845959595959, "avg_duration_per_task_ms": 643938.6085858586, "avg_output_tokens_per_task": 42440.19065656565, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.5851989731343283, "accuracy_std_dev": 0.25257942345483236, "run_count": 7, "total_questions": 1340, "avg_cost_per_task_usd": 0.0922404328358209, "avg_duration_per_task_ms": 1890268.2298507462, "avg_output_tokens_per_task": 40999.71268656717, "last_run_timestamp": "2026-10-31T16:02:18.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Fireworks", "accuracy": 0.5810130708860759, "accuracy_std_dev": 0.012019500000000016, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.01962629113924051, "avg_duration_per_task_ms": 376708.9949367089, "avg_output_tokens_per_task": 97953.7417721519, "last_run_timestamp": "2026-10-31T08:09:25.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5803875000000001, "accuracy_std_dev": 0.04095041650887082, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02901417803030303, "avg_duration_per_task_ms": 1378907.4116161617, "avg_output_tokens_per_task": 72425.33712121213, "last_run_timestamp": "2026-10-13T21:28:41.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.579686, "accuracy_std_dev": 0.008054024625407281, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02485565488215488, "avg_duration_per_task_ms": 1088356.1313131314, "avg_output_tokens_per_task": 66893.9074074074, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.5774414999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010289136363636365, "avg_duration_per_task_ms": 331432.74242424243, "avg_output_tokens_per_task": 40883.856060606064, "last_run_timestamp": "2026-10-17T16:06:05.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5764006434634974, "accuracy_std_dev": 0.0627820184359255, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.003614684210526316, "avg_duration_per_task_ms": 138004.24957555177, "avg_output_tokens_per_task": 3859.125636672326, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.5702345561377246, "accuracy_std_dev": 0.07436004550317579, "run_count": 8, "total_questions": 1336, "avg_cost_per_task_usd": 0.06098996482035929, "avg_duration_per_task_ms": 369346.24326347304, "avg_output_tokens_per_task": 23120.943113772457, "last_run_timestamp": "2026-10-31T16:21:16.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5628510000000001, "accuracy_std_dev": 0.07456152319170169, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005432700336700336, "avg_duration_per_task_ms": 118723.56734006734, "avg_output_tokens_per_task": 4911.058922558923, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.56060625, "accuracy_std_dev": 0.009297342103391676, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017428513888888886, "avg_duration_per_task_ms": 127446.60858585859, "avg_output_tokens_per_task": 43389.579545454544, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.55681825, "accuracy_std_dev": 0.011441432698202594, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004685652777777778, "avg_duration_per_task_ms": 548781.2613636364, "avg_output_tokens_per_task": 16501.491161616163, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5521885000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01778226515151515, "avg_duration_per_task_ms": 116667.00757575757, "avg_output_tokens_per_task": 44273.378787878784, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.5514771309523809, "accuracy_std_dev": 0.02267678126927133, "run_count": 8, "total_questions": 1512, "avg_cost_per_task_usd": 0.005655453042328042, "avg_duration_per_task_ms": 802376.1481481482, "avg_output_tokens_per_task": 14488.589285714286, "last_run_timestamp": "2026-11-04T18:37:19.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5420490229357798, "accuracy_std_dev": 0.06975384672944968, "run_count": 3, "total_questions": 218, "avg_cost_per_task_usd": 0.03485861009174312, "avg_duration_per_task_ms": 397279.0229357798, "avg_output_tokens_per_task": 16660.94495412844, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5373176666666667, "accuracy_std_dev": 0.013403444242266851, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0019087592592592591, "avg_duration_per_task_ms": 97924.95622895623, "avg_output_tokens_per_task": 1886.641414141414, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.53367, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004812772727272727, "avg_duration_per_task_ms": 168981.19191919192, "avg_output_tokens_per_task": 2773.419191919192, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.5266555, "accuracy_std_dev": 0.023895888382383546, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01547567760942761, "avg_duration_per_task_ms": 13716.89898989899, "avg_output_tokens_per_task": 1354.1801346801346, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.522278736040609, "accuracy_std_dev": 0.00921274336027126, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.016910294416243657, "avg_duration_per_task_ms": 664622.7783417936, "avg_output_tokens_per_task": 43236.38747884941, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.521886, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004345590909090909, "avg_duration_per_task_ms": 107214.82828282828, "avg_output_tokens_per_task": 2763.3333333333335, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.5216982222222223, "accuracy_std_dev": 0.1325981414523495, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.003293574635241302, "avg_duration_per_task_ms": 195367.39393939395, "avg_output_tokens_per_task": 7908.62962962963, "last_run_timestamp": "2026-10-31T16:10:36.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.510943, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013169762626262625, "avg_duration_per_task_ms": 12925.411616161617, "avg_output_tokens_per_task": 1121.608585858586, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004105035353535354, "avg_duration_per_task_ms": 27074.26767676768, "avg_output_tokens_per_task": 2644.373737373737, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0008276868686868686, "avg_duration_per_task_ms": 96676.70202020202, "avg_output_tokens_per_task": 2752.9141414141413, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.5050510000000001, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008273518518518518, "avg_duration_per_task_ms": 18715.93265993266, "avg_output_tokens_per_task": 1873.8602693602693, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.4995792500000001, "accuracy_std_dev": 0.02025051336849268, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008581388888888889, "avg_duration_per_task_ms": 12992.723484848484, "avg_output_tokens_per_task": 1853.0088383838383, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.49831649999999994, "accuracy_std_dev": 0.012284988196982519, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0023916161616161615, "avg_duration_per_task_ms": 9167.109217171717, "avg_output_tokens_per_task": 2380.501262626263, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.49242400000000003, "accuracy_std_dev": 0.0008419999999999817, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0017634166666666666, "avg_duration_per_task_ms": 30891.38888888889, "avg_output_tokens_per_task": 1731.7373737373737, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.49074049999999997, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008383863636363636, "avg_duration_per_task_ms": 22708.954545454544, "avg_output_tokens_per_task": 1902.2954545454545, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.490313482051282, "accuracy_std_dev": 0.025671764346586447, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.010895215384615384, "avg_duration_per_task_ms": 415746.58461538464, "avg_output_tokens_per_task": 27278.131623931626, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.486532, "accuracy_std_dev": 0.021886000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025879300505050506, "avg_duration_per_task_ms": 4030738.6136363638, "avg_output_tokens_per_task": 21539.305555555555, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.4851015826366559, "accuracy_std_dev": 0.011963524563621507, "run_count": 16, "total_questions": 3110, "avg_cost_per_task_usd": 0.0013306218649517683, "avg_duration_per_task_ms": 59520.47717041801, "avg_output_tokens_per_task": 2348.433118971061, "last_run_timestamp": "2026-10-31T16:01:53.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.483796, "accuracy_std_dev": 0.011661703413309726, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0025953238636363635, "avg_duration_per_task_ms": 48652.757575757576, "avg_output_tokens_per_task": 2847.4810606060605, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.48190212499999996, "accuracy_std_dev": 0.010369324127414242, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.002355522095959596, "avg_duration_per_task_ms": 15986.89898989899, "avg_output_tokens_per_task": 2214.277777777778, "last_run_timestamp": "2026-10-31T16:05:14.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.47966760309278356, "accuracy_std_dev": 0.006495603179587451, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.01110826116838488, "avg_duration_per_task_ms": 372116.1821305842, "avg_output_tokens_per_task": 27698.08591065292, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.47706212499999995, "accuracy_std_dev": 0.013881069307131027, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.001330816919191919, "avg_duration_per_task_ms": 45550.52525252525, "avg_output_tokens_per_task": 2388.7910353535353, "last_run_timestamp": "2026-10-31T16:36:11.519Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.47579937499999997, "accuracy_std_dev": 0.009595087765850562, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013503068181818184, "avg_duration_per_task_ms": 81673.49116161616, "avg_output_tokens_per_task": 2351.566919191919, "last_run_timestamp": "2026-10-31T16:23:41.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.475589, "accuracy_std_dev": 0.006204488912607278, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003132998316498317, "avg_duration_per_task_ms": 67179.39141414141, "avg_output_tokens_per_task": 2206.4680134680134, "last_run_timestamp": "2026-10-17T16:05:55.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.474747, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.04817519191919192, "avg_duration_per_task_ms": 40570.833333333336, "avg_output_tokens_per_task": 16881.383838383837, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47306366666666666, "accuracy_std_dev": 0.011903906959015132, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010633148148148147, "avg_duration_per_task_ms": 225625.12962962964, "avg_output_tokens_per_task": 2395.355218855219, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.472853375, "accuracy_std_dev": 0.009170414043780961, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0010971363636363637, "avg_duration_per_task_ms": 63777.9494949495, "avg_output_tokens_per_task": 2470.3926767676767, "last_run_timestamp": "2026-10-31T16:31:54.106Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.4662730708263069, "accuracy_std_dev": 0.0036637559355878987, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.0023418077571669476, "avg_duration_per_task_ms": 38653.266441821244, "avg_output_tokens_per_task": 2860.9241146711634, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4654881111111111, "accuracy_std_dev": 0.011153042389554961, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0008001296296296297, "avg_duration_per_task_ms": 150622.7519640853, "avg_output_tokens_per_task": 2249.726150392817, "last_run_timestamp": "2026-10-31T16:02:14.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.457071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0017076717171717172, "avg_duration_per_task_ms": 27081.62626262626, "avg_output_tokens_per_task": 1337.1363636363637, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.44871055972696244, "accuracy_std_dev": 0.08315656909813463, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.004654043230944255, "avg_duration_per_task_ms": 44940.08703071672, "avg_output_tokens_per_task": 1962.6114903299203, "last_run_timestamp": "2026-10-31T16:05:36.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.442761, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0013494898989898992, "avg_duration_per_task_ms": 69228.69696969698, "avg_output_tokens_per_task": 1528.5050505050506, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.43602700000000005, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010042146464646464, "avg_duration_per_task_ms": 21558.090909090908, "avg_output_tokens_per_task": 1481.030303030303, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.43139700000000003, "accuracy_std_dev": 0.012951914704011906, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010746982323232325, "avg_duration_per_task_ms": 24227.715909090908, "avg_output_tokens_per_task": 1509.2487373737374, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.429293, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0015700353535353536, "avg_duration_per_task_ms": 59122.85858585859, "avg_output_tokens_per_task": 1664.8737373737374, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4276093333333334, "accuracy_std_dev": 0.00629925838668507, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010292508417508418, "avg_duration_per_task_ms": 16725.542087542086, "avg_output_tokens_per_task": 1523.1902356902358, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.3855215, "accuracy_std_dev": 0.011293261320362695, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0001567070707070707, "avg_duration_per_task_ms": 7111.70202020202, "avg_output_tokens_per_task": 932.0808080808081, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.38047125000000004, "accuracy_std_dev": 0.012081350470353048, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0003843409090909091, "avg_duration_per_task_ms": 52179.61616161616, "avg_output_tokens_per_task": 2261.531565656566, "last_run_timestamp": "2026-11-01T11:50:48.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.34701175, "accuracy_std_dev": 0.09152431512547636, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008584518939393939, "avg_duration_per_task_ms": 487705.029040404, "avg_output_tokens_per_task": 21225.517045454544, "last_run_timestamp": "2026-10-31T16:45:48.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.29037048571428564, "accuracy_std_dev": 0.015956919278462715, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.002851500317460318, "avg_duration_per_task_ms": 40947.175873015876, "avg_output_tokens_per_task": 35610.56507936508, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2873176666666667, "accuracy_std_dev": 0.0007938452130121062, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006128619528619528, "avg_duration_per_task_ms": 10231.205387205388, "avg_output_tokens_per_task": 580.9023569023569, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.286195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005941969696969697, "avg_duration_per_task_ms": 14998.156565656565, "avg_output_tokens_per_task": 1642.1363636363637, "last_run_timestamp": "2026-10-13T18:56:05.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.27925087499999995, "accuracy_std_dev": 0.021335446769622016, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.001127453914141414, "avg_duration_per_task_ms": 148962.23926767678, "avg_output_tokens_per_task": 16512.824494949495, "last_run_timestamp": "2026-10-31T16:08:31.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.27665533333333336, "accuracy_std_dev": 0.0007938452130121063, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006201683501683503, "avg_duration_per_task_ms": 9496.395622895623, "avg_output_tokens_per_task": 580.2053872053872, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2745902896174863, "accuracy_std_dev": 0.009941000000000005, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.0000641912568306011, "avg_duration_per_task_ms": 8433.43169398907, "avg_output_tokens_per_task": 532.0136612021857, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.27161569453924916, "accuracy_std_dev": 0.020450238993441835, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.000055156996587030725, "avg_duration_per_task_ms": 10158.571672354949, "avg_output_tokens_per_task": 550.3515358361775, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.2711902753623188, "accuracy_std_dev": 0.014229495825133793, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.010431668642951253, "avg_duration_per_task_ms": 429362.8353096179, "avg_output_tokens_per_task": 46706.179183135704, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-4b-it", "model_slug": "google/gemma-3-4b-it", "model_name": "Google: Gemma 3 4B", "provider_name": "Parasail", "accuracy": 0.271044, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011306565656565657, "avg_duration_per_task_ms": 10846.11616161616, "avg_output_tokens_per_task": 1804.4646464646464, "last_run_timestamp": "2026-10-15T03:00:03.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.265993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007544898989898989, "avg_duration_per_task_ms": 47786.38888888889, "avg_output_tokens_per_task": 2583.777777777778, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.2644967777777778, "accuracy_std_dev": 0.010114842370142964, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0003960353535353535, "avg_duration_per_task_ms": 363717.83613916946, "avg_output_tokens_per_task": 9497.86251402918, "last_run_timestamp": "2026-10-31T16:08:31.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.256734125, "accuracy_std_dev": 0.014383764471075542, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00042851388888888887, "avg_duration_per_task_ms": 73900.38257575757, "avg_output_tokens_per_task": 8246.635732323231, "last_run_timestamp": "2026-10-31T16:55:07.715Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.157127, "accuracy_std_dev": 0.0633250504987297, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0016609797979797978, "avg_duration_per_task_ms": 58054.12962962963, "avg_output_tokens_per_task": 1445.550505050505, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.1409935, "accuracy_std_dev": 0.13931077227641084, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.000907405303030303, "avg_duration_per_task_ms": 385614.86616161617, "avg_output_tokens_per_task": 2945.2651515151515, "last_run_timestamp": "2026-10-24T16:03:19.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.013468000000000003, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006156141414141415, "avg_duration_per_task_ms": 168896.5202020202, "avg_output_tokens_per_task": 14359.661616161617, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.0025255000000000004, "accuracy_std_dev": 0.0008415000000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034195530303030306, "avg_duration_per_task_ms": 159765.32575757575, "avg_output_tokens_per_task": 3219.0782828282827, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.003405252525252525, "avg_duration_per_task_ms": 146900.18181818182, "avg_output_tokens_per_task": 3204.777777777778, "last_run_timestamp": "2026-10-31T16:04:43.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.82, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.8143706199999999, "avg_duration_per_task_ms": 141768.16, "avg_output_tokens_per_task": 5650.52, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.816667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05960144, "avg_duration_per_task_ms": 162346.76, "avg_output_tokens_per_task": 42454.8, "last_run_timestamp": "2026-10-17T19:19:25.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.80208325, "accuracy_std_dev": 0.012126500481899135, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1338215925, "avg_duration_per_task_ms": 2777506.835, "avg_output_tokens_per_task": 27899.5725, "last_run_timestamp": "2026-10-31T20:31:41.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8000000909090909, "accuracy_std_dev": 0.020695063581156234, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8742483381818181, "avg_duration_per_task_ms": 136193.65272727274, "avg_output_tokens_per_task": 5706.489090909091, "last_run_timestamp": "2026-10-31T12:24:55.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.8, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.39313749000000003, "avg_duration_per_task_ms": 477750.16, "avg_output_tokens_per_task": 34082.08, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.7981818181818182, "accuracy_std_dev": 0.021851955343414428, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8809048418181818, "avg_duration_per_task_ms": 125745.05454545455, "avg_output_tokens_per_task": 5732.8836363636365, "last_run_timestamp": "2026-10-31T11:20:38.480Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.7981221830985916, "accuracy_std_dev": 0.05639978873679898, "run_count": 3, "total_questions": 142, "avg_cost_per_task_usd": 0.007624964788732395, "avg_duration_per_task_ms": 30939.802816901407, "avg_output_tokens_per_task": 8761.732394366198, "last_run_timestamp": "2026-10-31T11:56:20.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7956450603015076, "accuracy_std_dev": 0.0288553624912251, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.2041434145728643, "avg_duration_per_task_ms": 814395.7738693467, "avg_output_tokens_per_task": 35286.695979899494, "last_run_timestamp": "2026-10-31T19:01:09.612Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.7939393636363636, "accuracy_std_dev": 0.02791815599951441, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8490899109090908, "avg_duration_per_task_ms": 118340.14, "avg_output_tokens_per_task": 5735.276363636363, "last_run_timestamp": "2026-10-31T11:38:36.784Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7933333636363636, "accuracy_std_dev": 0.01705605730990246, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8212979, "avg_duration_per_task_ms": 123271.04, "avg_output_tokens_per_task": 5720.44, "last_run_timestamp": "2026-10-31T11:18:52.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00828808, "avg_duration_per_task_ms": 196026.56, "avg_output_tokens_per_task": 16303, "last_run_timestamp": "2026-10-14T19:21:35.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03146312, "avg_duration_per_task_ms": 636210.72, "avg_output_tokens_per_task": 16745.44, "last_run_timestamp": "2026-10-14T19:17:59.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7901233981481481, "accuracy_std_dev": 0.021051257864696603, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.07926520925925926, "avg_duration_per_task_ms": 1152985.5148148148, "avg_output_tokens_per_task": 25849.22962962963, "last_run_timestamp": "2026-10-31T20:22:41.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7896970909090909, "accuracy_std_dev": 0.02153015652712829, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.8954311818181818, "avg_duration_per_task_ms": 92327.7890909091, "avg_output_tokens_per_task": 5722.323636363636, "last_run_timestamp": "2026-10-31T12:30:34.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.789473611528822, "accuracy_std_dev": 0.025156094260387483, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.18347024812030074, "avg_duration_per_task_ms": 432487.7944862155, "avg_output_tokens_per_task": 30439.06015037594, "last_run_timestamp": "2026-10-31T19:12:41.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.788889, "accuracy_std_dev": 0.015234869040023505, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.057788776666666666, "avg_duration_per_task_ms": 893038.1233333333, "avg_output_tokens_per_task": 40447.416666666664, "last_run_timestamp": "2026-10-31T13:13:33.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7882883714036617, "accuracy_std_dev": 0.015477690995371508, "run_count": 23, "total_questions": 1147, "avg_cost_per_task_usd": 0.0821072258064516, "avg_duration_per_task_ms": 149668.3635571055, "avg_output_tokens_per_task": 12209.048823016565, "last_run_timestamp": "2026-10-31T12:32:50.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Tenstorrent", "accuracy": 0.788194, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.0031695833333333333, "avg_duration_per_task_ms": 66197.89583333333, "avg_output_tokens_per_task": 2922.6041666666665, "last_run_timestamp": "2026-11-04T18:22:51.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.788, "accuracy_std_dev": 0.02246951515275753, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.15925035199999998, "avg_duration_per_task_ms": 71120.884, "avg_output_tokens_per_task": 6245.168, "last_run_timestamp": "2026-10-31T11:10:36.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7874073333333333, "accuracy_std_dev": 0.02836162518850661, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08114153333333332, "avg_duration_per_task_ms": 209436.06444444443, "avg_output_tokens_per_task": 25962.27111111111, "last_run_timestamp": "2026-10-31T19:22:16.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.49828548, "avg_duration_per_task_ms": 348417.68, "avg_output_tokens_per_task": 21971.04, "last_run_timestamp": "2026-10-14T15:16:05.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Fireworks", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20422622, "avg_duration_per_task_ms": 176471.64, "avg_output_tokens_per_task": 10146.04, "last_run_timestamp": "2026-11-05T15:41:01.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7866666363636363, "accuracy_std_dev": 0.020695005011191054, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4729785890909092, "avg_duration_per_task_ms": 117016.24545454545, "avg_output_tokens_per_task": 6994.309090909091, "last_run_timestamp": "2026-10-31T12:17:12.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.786127239884393, "accuracy_std_dev": 0.023660230134130145, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.09665253179190753, "avg_duration_per_task_ms": 380894.4450867052, "avg_output_tokens_per_task": 16737.754335260117, "last_run_timestamp": "2026-10-31T12:56:48.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7859650631578948, "accuracy_std_dev": 0.03135199999999999, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.09807105263157893, "avg_duration_per_task_ms": 69953.93684210526, "avg_output_tokens_per_task": 8018.557894736842, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.7858794166666667, "accuracy_std_dev": 0.021093449409710124, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.07943493055555556, "avg_duration_per_task_ms": 334937.25694444444, "avg_output_tokens_per_task": 18674.54861111111, "last_run_timestamp": "2026-10-31T19:43:30.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.785, "accuracy_std_dev": 0.023979192370678373, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.20099723, "avg_duration_per_task_ms": 767265.06, "avg_output_tokens_per_task": 32060.4625, "last_run_timestamp": "2026-10-31T19:47:27.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.7844443333333334, "accuracy_std_dev": 0.01632984089872824, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0785912688888889, "avg_duration_per_task_ms": 47362.52222222222, "avg_output_tokens_per_task": 4720.888888888889, "last_run_timestamp": "2026-10-31T11:58:26.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.7833333333333333, "accuracy_std_dev": 0.011706202852429232, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.1729425, "avg_duration_per_task_ms": 66541.35, "avg_output_tokens_per_task": 6203.445, "last_run_timestamp": "2026-10-31T12:38:55.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7833330000000001, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07443635, "avg_duration_per_task_ms": 98397.15, "avg_output_tokens_per_task": 21518.67, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7831052054794521, "accuracy_std_dev": 0.017155537965592597, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.14992966438356164, "avg_duration_per_task_ms": 183129.3904109589, "avg_output_tokens_per_task": 16313.164383561643, "last_run_timestamp": "2026-10-15T21:26:28.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7825, "accuracy_std_dev": 0.023790007240436065, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08953825, "avg_duration_per_task_ms": 274829.5175, "avg_output_tokens_per_task": 19992.6725, "last_run_timestamp": "2026-10-31T12:57:09.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7822223333333334, "accuracy_std_dev": 0.01571327975171178, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.011317603333333334, "avg_duration_per_task_ms": 599501.3033333333, "avg_output_tokens_per_task": 21464.33, "last_run_timestamp": "2026-10-31T14:08:11.994Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.781333, "accuracy_std_dev": 0.027129319932501044, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0587014, "avg_duration_per_task_ms": 871009.496, "avg_output_tokens_per_task": 40629.384, "last_run_timestamp": "2026-10-31T13:41:45.309Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7811976407035177, "accuracy_std_dev": 0.02110393286871771, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.09273568467336683, "avg_duration_per_task_ms": 66515.5364321608, "avg_output_tokens_per_task": 4679.810301507538, "last_run_timestamp": "2026-10-31T13:12:01.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.7809525714285713, "accuracy_std_dev": 0.03331975778019121, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04230577142857143, "avg_duration_per_task_ms": 122423.61428571428, "avg_output_tokens_per_task": 4845.648571428572, "last_run_timestamp": "2026-11-04T18:25:27.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7808333749999999, "accuracy_std_dev": 0.010769114145758466, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0823930725, "avg_duration_per_task_ms": 41435.8875, "avg_output_tokens_per_task": 4744.8375, "last_run_timestamp": "2026-10-31T12:40:46.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7800001, "accuracy_std_dev": 0.02108170349592273, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.174296886, "avg_duration_per_task_ms": 72081.318, "avg_output_tokens_per_task": 6231.684, "last_run_timestamp": "2026-10-31T12:10:32.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.78, "accuracy_std_dev": 0.021081938910409114, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.18010515333333332, "avg_duration_per_task_ms": 64530.58666666667, "avg_output_tokens_per_task": 6325.823333333334, "last_run_timestamp": "2026-10-31T12:07:29.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "PrimeIntellect", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0042799399999999994, "avg_duration_per_task_ms": 627315.22, "avg_output_tokens_per_task": 12675.1, "last_run_timestamp": "2026-10-31T21:10:18.835Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7790587291066282, "accuracy_std_dev": 0.03375729268813859, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.19038813832853027, "avg_duration_per_task_ms": 636053.7665706052, "avg_output_tokens_per_task": 30091.458213256483, "last_run_timestamp": "2026-10-31T19:27:20.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7786668, "accuracy_std_dev": 0.01484714413481597, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014840148000000001, "avg_duration_per_task_ms": 379677.016, "avg_output_tokens_per_task": 21736.348, "last_run_timestamp": "2026-10-31T12:28:58.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.7786666, "accuracy_std_dev": 0.010666575000439441, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.031286299999999996, "avg_duration_per_task_ms": 221994.964, "avg_output_tokens_per_task": 12740.492, "last_run_timestamp": "2026-10-31T12:48:16.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7784256831875607, "accuracy_std_dev": 0.029691712716069543, "run_count": 21, "total_questions": 1029, "avg_cost_per_task_usd": 0.07680297376093294, "avg_duration_per_task_ms": 74568.79203109816, "avg_output_tokens_per_task": 4283.289601554908, "last_run_timestamp": "2026-10-31T13:24:01.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.77833325, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12165100000000001, "avg_duration_per_task_ms": 123453.655, "avg_output_tokens_per_task": 4597.39, "last_run_timestamp": "2026-10-28T05:13:02.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7779999000000001, "accuracy_std_dev": 0.020011142453393298, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.081736628, "avg_duration_per_task_ms": 135707.96, "avg_output_tokens_per_task": 4671.7, "last_run_timestamp": "2026-10-31T12:38:49.692Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.777778, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.016507473333333335, "avg_duration_per_task_ms": 92510.76666666666, "avg_output_tokens_per_task": 10861.266666666666, "last_run_timestamp": "2026-10-31T11:43:38.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7777776666666667, "accuracy_std_dev": 0.01791624892071129, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4406913983333334, "avg_duration_per_task_ms": 87633.54833333334, "avg_output_tokens_per_task": 5698.321666666667, "last_run_timestamp": "2026-10-31T12:26:17.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.777459306590258, "accuracy_std_dev": 0.02712463143584833, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.17213766475644698, "avg_duration_per_task_ms": 55665.0888252149, "avg_output_tokens_per_task": 4643.535816618911, "last_run_timestamp": "2026-10-31T12:26:01.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7773700055045871, "accuracy_std_dev": 0.023259459839311162, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.19957610458715597, "avg_duration_per_task_ms": 504017.81834862387, "avg_output_tokens_per_task": 27873.480733944954, "last_run_timestamp": "2026-10-31T18:53:29.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7769697272727273, "accuracy_std_dev": 0.023503914842545274, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.17000764545454541, "avg_duration_per_task_ms": 562646.2527272727, "avg_output_tokens_per_task": 35727.95818181818, "last_run_timestamp": "2026-10-31T19:08:11.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.014782246611692315, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17363205, "avg_duration_per_task_ms": 73619.57333333333, "avg_output_tokens_per_task": 6340.086666666667, "last_run_timestamp": "2026-10-31T12:24:45.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7766665833333334, "accuracy_std_dev": 0.0379571517689318, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10135336166666668, "avg_duration_per_task_ms": 172908.82, "avg_output_tokens_per_task": 6639.301666666666, "last_run_timestamp": "2026-10-31T11:53:06.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7764815, "accuracy_std_dev": 0.02583109973193382, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.13240519666666667, "avg_duration_per_task_ms": 319498.1977777778, "avg_output_tokens_per_task": 19319.296666666665, "last_run_timestamp": "2026-10-31T13:25:38.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.7760940000000001, "accuracy_std_dev": 0.020936325423770052, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.06970348989898989, "avg_duration_per_task_ms": 95095.22222222222, "avg_output_tokens_per_task": 4292.19696969697, "last_run_timestamp": "2026-10-31T12:44:06.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.7760001999999999, "accuracy_std_dev": 0.028472283599318148, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06820797999999999, "avg_duration_per_task_ms": 658315.768, "avg_output_tokens_per_task": 13133.756, "last_run_timestamp": "2026-10-31T13:30:16.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7758335, "accuracy_std_dev": 0.016308646311082982, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.15398643500000003, "avg_duration_per_task_ms": 330861.565, "avg_output_tokens_per_task": 19948.0325, "last_run_timestamp": "2026-10-31T13:41:05.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7757695461847389, "accuracy_std_dev": 0.016169521055368324, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.08735203012048193, "avg_duration_per_task_ms": 139136.04417670684, "avg_output_tokens_per_task": 4443.712851405622, "last_run_timestamp": "2026-10-31T12:29:23.419Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7757574545454545, "accuracy_std_dev": 0.02074823590031709, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08103076363636365, "avg_duration_per_task_ms": 336159.44, "avg_output_tokens_per_task": 20208.643636363635, "last_run_timestamp": "2026-10-31T12:24:53.587Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7753625384615385, "accuracy_std_dev": 0.017323958686038136, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06710363879598663, "avg_duration_per_task_ms": 1125092.9698996656, "avg_output_tokens_per_task": 36573.67558528428, "last_run_timestamp": "2026-10-24T13:26:35.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7753334, "accuracy_std_dev": 0.026339740994170767, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.065886822, "avg_duration_per_task_ms": 257998.344, "avg_output_tokens_per_task": 14164.592, "last_run_timestamp": "2026-10-31T12:39:11.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7752688387096777, "accuracy_std_dev": 0.0207523765107747, "run_count": 31, "total_questions": 1550, "avg_cost_per_task_usd": 0.4981299012903227, "avg_duration_per_task_ms": 112970.55806451612, "avg_output_tokens_per_task": 7038.44064516129, "last_run_timestamp": "2026-10-31T11:36:20.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7751514545454545, "accuracy_std_dev": 0.026375843918816165, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10104016181818182, "avg_duration_per_task_ms": 294074.4890909091, "avg_output_tokens_per_task": 21520.51818181818, "last_run_timestamp": "2026-10-31T11:29:11.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7750724782608696, "accuracy_std_dev": 0.020450702198786246, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.47903272521739126, "avg_duration_per_task_ms": 101622.89130434782, "avg_output_tokens_per_task": 7123.294782608696, "last_run_timestamp": "2026-10-31T12:54:18.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.775, "accuracy_std_dev": 0.01787294665828779, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09322250500000001, "avg_duration_per_task_ms": 167383.47, "avg_output_tokens_per_task": 5123.47, "last_run_timestamp": "2026-10-31T12:32:51.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7745454545454545, "accuracy_std_dev": 0.02425759601131023, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.48932630545454553, "avg_duration_per_task_ms": 124123.69272727273, "avg_output_tokens_per_task": 7977.827272727272, "last_run_timestamp": "2026-10-31T11:58:26.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7743749687500002, "accuracy_std_dev": 0.022737010850269506, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.49075661062500003, "avg_duration_per_task_ms": 106422.6925, "avg_output_tokens_per_task": 7062.208125, "last_run_timestamp": "2026-10-31T12:26:08.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.026945723560596418, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21623159142857146, "avg_duration_per_task_ms": 562306.5571428571, "avg_output_tokens_per_task": 12602.945714285714, "last_run_timestamp": "2026-10-10T11:06:43.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7739394545454545, "accuracy_std_dev": 0.019165412105066386, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06798592909090909, "avg_duration_per_task_ms": 45561.432727272724, "avg_output_tokens_per_task": 4731.818181818182, "last_run_timestamp": "2026-10-31T12:41:41.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7733334166666667, "accuracy_std_dev": 0.024343346495015628, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.506005475, "avg_duration_per_task_ms": 120787.225, "avg_output_tokens_per_task": 7991.263333333333, "last_run_timestamp": "2026-10-31T11:50:39.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7733334, "accuracy_std_dev": 0.02108173511929225, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.05645720600000001, "avg_duration_per_task_ms": 156313.234, "avg_output_tokens_per_task": 16426.692, "last_run_timestamp": "2026-10-31T11:34:45.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.773333375, "accuracy_std_dev": 0.02236071704428047, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04385152750000001, "avg_duration_per_task_ms": 107910.09, "avg_output_tokens_per_task": 4940.6725, "last_run_timestamp": "2026-10-31T12:00:41.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.02309381831909905, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.16708652333333332, "avg_duration_per_task_ms": 79748.64333333333, "avg_output_tokens_per_task": 6314.02, "last_run_timestamp": "2026-10-31T11:48:36.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.04109609335312651, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08230668000000001, "avg_duration_per_task_ms": 584422.56, "avg_output_tokens_per_task": 10999.886666666667, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.7733332857142857, "accuracy_std_dev": 0.022537305934283946, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08668051142857143, "avg_duration_per_task_ms": 364495.4428571428, "avg_output_tokens_per_task": 20483.94, "last_run_timestamp": "2026-10-24T12:45:56.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.77333325, "accuracy_std_dev": 0.029814193116492343, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06547900499999999, "avg_duration_per_task_ms": 79239.345, "avg_output_tokens_per_task": 12411.3875, "last_run_timestamp": "2026-10-31T11:09:23.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7733331666666667, "accuracy_std_dev": 0.021081851068448003, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05726335, "avg_duration_per_task_ms": 158508.54333333333, "avg_output_tokens_per_task": 4588.7266666666665, "last_run_timestamp": "2026-10-31T13:11:06.716Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06913468, "avg_duration_per_task_ms": 170689.28, "avg_output_tokens_per_task": 8815.54, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Relace", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14084568, "avg_duration_per_task_ms": 130912.8, "avg_output_tokens_per_task": 10689.44, "last_run_timestamp": "2026-11-05T22:50:26.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Nebius", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2468412, "avg_duration_per_task_ms": 828355.82, "avg_output_tokens_per_task": 10570.46, "last_run_timestamp": "2026-11-04T22:42:55.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11485508, "avg_duration_per_task_ms": 381375.48, "avg_output_tokens_per_task": 14199.1, "last_run_timestamp": "2026-10-13T03:50:19.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7725926666666666, "accuracy_std_dev": 0.033176625054396364, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05704026, "avg_duration_per_task_ms": 115075.10666666667, "avg_output_tokens_per_task": 6558.98, "last_run_timestamp": "2026-10-31T12:50:03.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7722222222222223, "accuracy_std_dev": 0.03095202181433487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03593436666666667, "avg_duration_per_task_ms": 446735.1666666667, "avg_output_tokens_per_task": 13815.786666666667, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7721739130434783, "accuracy_std_dev": 0.03024227286994588, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.5151446756521739, "avg_duration_per_task_ms": 124707.92, "avg_output_tokens_per_task": 7934.98347826087, "last_run_timestamp": "2026-10-31T13:09:20.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7721212727272727, "accuracy_std_dev": 0.023236674253863547, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.14458932363636365, "avg_duration_per_task_ms": 821555.8181818182, "avg_output_tokens_per_task": 31749.51818181818, "last_run_timestamp": "2026-10-31T20:06:11.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.7720001999999999, "accuracy_std_dev": 0.024367767418456705, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.14564374000000002, "avg_duration_per_task_ms": 314489.068, "avg_output_tokens_per_task": 20798.24, "last_run_timestamp": "2026-10-31T13:16:57.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7718182727272727, "accuracy_std_dev": 0.031151820244762325, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06332707454545455, "avg_duration_per_task_ms": 243359.18363636362, "avg_output_tokens_per_task": 11661.423636363636, "last_run_timestamp": "2026-10-31T12:57:09.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.771666625, "accuracy_std_dev": 0.02127857229666442, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.058498104999999995, "avg_duration_per_task_ms": 220433.295, "avg_output_tokens_per_task": 12747.45, "last_run_timestamp": "2026-10-31T20:15:14.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.7716665, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.050993250000000004, "avg_duration_per_task_ms": 677393.46, "avg_output_tokens_per_task": 30421.97, "last_run_timestamp": "2026-10-31T13:28:05.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7715774062499999, "accuracy_std_dev": 0.022644299042712767, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.14963372544642858, "avg_duration_per_task_ms": 341867.6473214286, "avg_output_tokens_per_task": 20209.140625, "last_run_timestamp": "2026-10-31T14:04:49.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7715151818181818, "accuracy_std_dev": 0.01635249901846076, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4578688290909091, "avg_duration_per_task_ms": 118180.02, "avg_output_tokens_per_task": 7063.84, "last_run_timestamp": "2026-10-31T11:40:34.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7714602976588629, "accuracy_std_dev": 0.028416062527791094, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.08344332775919731, "avg_duration_per_task_ms": 249520.2474916388, "avg_output_tokens_per_task": 12314.311036789297, "last_run_timestamp": "2026-10-31T19:22:52.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7713334, "accuracy_std_dev": 0.020011225740568715, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.051776854, "avg_duration_per_task_ms": 108112.036, "avg_output_tokens_per_task": 4665.94, "last_run_timestamp": "2026-10-31T13:28:16.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7709090909090909, "accuracy_std_dev": 0.015380107124905002, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.43817922727272723, "avg_duration_per_task_ms": 92556, "avg_output_tokens_per_task": 5682.427272727273, "last_run_timestamp": "2026-10-31T12:52:03.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.770833375, "accuracy_std_dev": 0.02283704805758343, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2110248925, "avg_duration_per_task_ms": 381381.2675, "avg_output_tokens_per_task": 17629.1275, "last_run_timestamp": "2026-10-31T12:14:30.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7706668, "accuracy_std_dev": 0.027194791985966728, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.13800210599999999, "avg_duration_per_task_ms": 324537.226, "avg_output_tokens_per_task": 20086.83, "last_run_timestamp": "2026-10-31T13:20:51.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.7702915342789598, "accuracy_std_dev": 0.030639008330912567, "run_count": 17, "total_questions": 846, "avg_cost_per_task_usd": 0.034585855791962175, "avg_duration_per_task_ms": 444223.6111111111, "avg_output_tokens_per_task": 12313.30023640662, "last_run_timestamp": "2026-10-31T19:33:25.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7700000714285714, "accuracy_std_dev": 0.014638549891415803, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.14041841571428573, "avg_duration_per_task_ms": 235377.64428571428, "avg_output_tokens_per_task": 25220.13142857143, "last_run_timestamp": "2026-10-31T13:20:49.855Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.35129818, "avg_duration_per_task_ms": 177012.35, "avg_output_tokens_per_task": 12679.64, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.77, "accuracy_std_dev": 0.019051314722786625, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01876148, "avg_duration_per_task_ms": 512061.9, "avg_output_tokens_per_task": 12105.946666666667, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.77, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07660454, "avg_duration_per_task_ms": 184215.64, "avg_output_tokens_per_task": 8506.78, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Moonshot AI", "accuracy": 0.77, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28240516, "avg_duration_per_task_ms": 307072.21, "avg_output_tokens_per_task": 10658.56, "last_run_timestamp": "2026-11-04T18:56:58.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.769999875, "accuracy_std_dev": 0.03382961802340332, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0406579, "avg_duration_per_task_ms": 682767.975, "avg_output_tokens_per_task": 12586.08, "last_run_timestamp": "2026-10-31T12:34:48.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7699214781144781, "accuracy_std_dev": 0.02849514533750542, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.010507353535353536, "avg_duration_per_task_ms": 977066.3771043771, "avg_output_tokens_per_task": 18208.95286195286, "last_run_timestamp": "2026-10-31T12:24:58.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7696968636363637, "accuracy_std_dev": 0.030466247715218248, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.4521496381818182, "avg_duration_per_task_ms": 78848.03, "avg_output_tokens_per_task": 5649.991818181818, "last_run_timestamp": "2026-10-31T12:41:58.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.7696666, "accuracy_std_dev": 0.019913617899317042, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.07924518799999998, "avg_duration_per_task_ms": 90727.314, "avg_output_tokens_per_task": 4394.952, "last_run_timestamp": "2026-11-05T00:52:28.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.7695237142857143, "accuracy_std_dev": 0.02386662296246901, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11461242285714283, "avg_duration_per_task_ms": 472929.0742857143, "avg_output_tokens_per_task": 11699.722857142857, "last_run_timestamp": "2026-10-23T07:26:58.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7693333, "accuracy_std_dev": 0.027680754646685506, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.080204858, "avg_duration_per_task_ms": 55518.588, "avg_output_tokens_per_task": 4833.54, "last_run_timestamp": "2026-10-31T12:18:44.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.769090909090909, "accuracy_std_dev": 0.02562719156909333, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4768810909090909, "avg_duration_per_task_ms": 108340.30181818182, "avg_output_tokens_per_task": 7142.923636363636, "last_run_timestamp": "2026-10-31T11:27:21.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7690024251781471, "accuracy_std_dev": 0.02344176596675554, "run_count": 17, "total_questions": 842, "avg_cost_per_task_usd": 0.05268543230403801, "avg_duration_per_task_ms": 269011.41448931117, "avg_output_tokens_per_task": 11083.264845605701, "last_run_timestamp": "2026-10-31T12:29:57.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "PrimaLabs", "accuracy": 0.7689656482758621, "accuracy_std_dev": 0.012187977929090621, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.5075254827586206, "avg_duration_per_task_ms": 227003.33793103448, "avg_output_tokens_per_task": 39823.972413793104, "last_run_timestamp": "2026-11-06T16:54:34.176Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7688888333333334, "accuracy_std_dev": 0.013698652724710655, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15528754666666666, "avg_duration_per_task_ms": 428069.5, "avg_output_tokens_per_task": 20104.073333333334, "last_run_timestamp": "2026-10-31T20:34:04.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.03456950772316872, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07998294666666667, "avg_duration_per_task_ms": 191648.96, "avg_output_tokens_per_task": 9368.36, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7684848181818182, "accuracy_std_dev": 0.034709847551636586, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05697716181818182, "avg_duration_per_task_ms": 301468.96545454545, "avg_output_tokens_per_task": 12590.405454545455, "last_run_timestamp": "2026-10-31T11:18:52.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Phala", "accuracy": 0.7683335, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.18849928, "avg_duration_per_task_ms": 346868.4, "avg_output_tokens_per_task": 10339.12, "last_run_timestamp": "2026-11-05T18:26:07.849Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7683333999999999, "accuracy_std_dev": 0.023154082915114577, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.127404986, "avg_duration_per_task_ms": 482373.346, "avg_output_tokens_per_task": 21637.406, "last_run_timestamp": "2026-11-05T02:16:26.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.768333375, "accuracy_std_dev": 0.026193847349795237, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0433664925, "avg_duration_per_task_ms": 202126.965, "avg_output_tokens_per_task": 13303.9375, "last_run_timestamp": "2026-10-31T20:06:41.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.768333375, "accuracy_std_dev": 0.027638527357375157, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12531909, "avg_duration_per_task_ms": 166385.7025, "avg_output_tokens_per_task": 7956.0025, "last_run_timestamp": "2026-10-31T13:24:47.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7681482222222222, "accuracy_std_dev": 0.017435059205697385, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.052077991111111104, "avg_duration_per_task_ms": 111760.7, "avg_output_tokens_per_task": 4431.0888888888885, "last_run_timestamp": "2026-10-31T13:22:10.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7681249687499998, "accuracy_std_dev": 0.028172817762078275, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.5299744849999999, "avg_duration_per_task_ms": 112471.520625, "avg_output_tokens_per_task": 8064.054375, "last_run_timestamp": "2026-10-31T12:31:31.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.767879, "accuracy_std_dev": 0.0204628104530234, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4928555472727273, "avg_duration_per_task_ms": 123502.80727272727, "avg_output_tokens_per_task": 8045.849090909091, "last_run_timestamp": "2026-10-31T12:30:31.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7677776666666667, "accuracy_std_dev": 0.043104839177934014, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06842919666666666, "avg_duration_per_task_ms": 974829.46, "avg_output_tokens_per_task": 44158.48333333333, "last_run_timestamp": "2026-10-31T13:18:29.756Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.767500125, "accuracy_std_dev": 0.028369943260594926, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.141204825, "avg_duration_per_task_ms": 285064.7125, "avg_output_tokens_per_task": 12155.7, "last_run_timestamp": "2026-10-31T19:58:31.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7675, "accuracy_std_dev": 0.019561171859068173, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0578507225, "avg_duration_per_task_ms": 50974.42, "avg_output_tokens_per_task": 4677.1725, "last_run_timestamp": "2026-10-24T12:36:03.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7674509411764706, "accuracy_std_dev": 0.021678839985437112, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.12603030588235295, "avg_duration_per_task_ms": 92367.94588235294, "avg_output_tokens_per_task": 4470.871764705882, "last_run_timestamp": "2026-10-31T12:50:48.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7674074444444444, "accuracy_std_dev": 0.02972210199726465, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08416672666666668, "avg_duration_per_task_ms": 371249.3911111111, "avg_output_tokens_per_task": 18973.266666666666, "last_run_timestamp": "2026-10-31T20:27:36.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7672726363636364, "accuracy_std_dev": 0.030278820027065213, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.43524433636363635, "avg_duration_per_task_ms": 90898.93818181819, "avg_output_tokens_per_task": 5582.549090909091, "last_run_timestamp": "2026-10-31T12:33:27.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7670707272727272, "accuracy_std_dev": 0.02101397565441647, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.06510748121212123, "avg_duration_per_task_ms": 67556.39090909091, "avg_output_tokens_per_task": 4633.900606060606, "last_run_timestamp": "2026-11-02T21:36:47.807Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7666669999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2750355, "avg_duration_per_task_ms": 98712.99, "avg_output_tokens_per_task": 5708.24, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01300872, "avg_duration_per_task_ms": 90405.02, "avg_output_tokens_per_task": 13808.24, "last_run_timestamp": "2026-10-29T03:35:28.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0213333, "avg_duration_per_task_ms": 951098.34, "avg_output_tokens_per_task": 12473.42, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Nebius", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0734938, "avg_duration_per_task_ms": 215054.84, "avg_output_tokens_per_task": 13096.66, "last_run_timestamp": "2026-11-04T18:29:36.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7666668, "accuracy_std_dev": 0.007302967435227963, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.220517392, "avg_duration_per_task_ms": 158612.068, "avg_output_tokens_per_task": 23228.836, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031067933333333332, "avg_duration_per_task_ms": 60278.37, "avg_output_tokens_per_task": 4752.3966666666665, "last_run_timestamp": "2026-10-31T12:25:28.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.023517736937610882, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04758358666666667, "avg_duration_per_task_ms": 110427.16444444444, "avg_output_tokens_per_task": 4684.946666666667, "last_run_timestamp": "2026-10-31T12:08:49.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7666666363636364, "accuracy_std_dev": 0.038454932300439754, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08742998, "avg_duration_per_task_ms": 299403.17272727273, "avg_output_tokens_per_task": 15964.536363636364, "last_run_timestamp": "2026-10-31T19:58:49.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.27949224, "avg_duration_per_task_ms": 363191.33, "avg_output_tokens_per_task": 35460.86, "last_run_timestamp": "2026-10-15T21:31:19.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.7666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01352679, "avg_duration_per_task_ms": 90848.6, "avg_output_tokens_per_task": 4507.38, "last_run_timestamp": "2026-10-10T11:18:50.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.766068730941704, "accuracy_std_dev": 0.02506483163840778, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.06749501121076233, "avg_duration_per_task_ms": 62014.139013452914, "avg_output_tokens_per_task": 4517.719730941704, "last_run_timestamp": "2026-10-31T13:21:43.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.7660001999999999, "accuracy_std_dev": 0.0260256591340162, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.12242998000000002, "avg_duration_per_task_ms": 384915.524, "avg_output_tokens_per_task": 20927.716, "last_run_timestamp": "2026-10-31T13:14:15.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7658823529411766, "accuracy_std_dev": 0.020244567150490694, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.5334031917647061, "avg_duration_per_task_ms": 118335.29588235295, "avg_output_tokens_per_task": 8074.314705882353, "last_run_timestamp": "2026-10-31T12:26:06.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.765833375, "accuracy_std_dev": 0.019843192579178773, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1200979175, "avg_duration_per_task_ms": 261332.9925, "avg_output_tokens_per_task": 20111.6625, "last_run_timestamp": "2026-10-31T19:46:00.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7655556666666666, "accuracy_std_dev": 0.02746477429767488, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5335597466666667, "avg_duration_per_task_ms": 180574.04166666666, "avg_output_tokens_per_task": 10505.758333333333, "last_run_timestamp": "2026-10-31T12:38:33.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.7653333999999999, "accuracy_std_dev": 0.023999911113168737, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.088749776, "avg_duration_per_task_ms": 386636.328, "avg_output_tokens_per_task": 14600.352, "last_run_timestamp": "2026-10-31T12:27:43.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7652380952380953, "accuracy_std_dev": 0.027344026724583557, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.044898270476190466, "avg_duration_per_task_ms": 87109.40476190476, "avg_output_tokens_per_task": 4336.9914285714285, "last_run_timestamp": "2026-11-06T23:32:54.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7648888, "accuracy_std_dev": 0.023153787094699365, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.03474511733333333, "avg_duration_per_task_ms": 66218.656, "avg_output_tokens_per_task": 4837.296, "last_run_timestamp": "2026-11-04T21:48:00.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7648485454545456, "accuracy_std_dev": 0.020842423656674335, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.5573711296969697, "avg_duration_per_task_ms": 70870.49272727272, "avg_output_tokens_per_task": 5721.953939393939, "last_run_timestamp": "2026-10-31T12:33:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.04565023765059233, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01549936, "avg_duration_per_task_ms": 297786.8, "avg_output_tokens_per_task": 16565.086666666666, "last_run_timestamp": "2026-10-28T04:10:41.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.76439777486911, "accuracy_std_dev": 0.015263579494993957, "run_count": 4, "total_questions": 191, "avg_cost_per_task_usd": 0.06284435078534031, "avg_duration_per_task_ms": 156780.0209424084, "avg_output_tokens_per_task": 9570.141361256545, "last_run_timestamp": "2026-10-31T19:43:06.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7642425454545454, "accuracy_std_dev": 0.021324467855935902, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.42425751818181817, "avg_duration_per_task_ms": 89884.58, "avg_output_tokens_per_task": 5563.18, "last_run_timestamp": "2026-10-31T11:48:47.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7642422727272727, "accuracy_std_dev": 0.025309747612300426, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02170633272727273, "avg_duration_per_task_ms": 276388.2545454545, "avg_output_tokens_per_task": 15679.149090909092, "last_run_timestamp": "2026-10-31T13:19:25.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.7641668749999999, "accuracy_std_dev": 0.026233564961883768, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.3343550775, "avg_duration_per_task_ms": 660538.1825, "avg_output_tokens_per_task": 28260.275, "last_run_timestamp": "2026-10-31T19:49:31.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7641665000000001, "accuracy_std_dev": 0.021586383312171588, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0185151775, "avg_duration_per_task_ms": 270096.6575, "avg_output_tokens_per_task": 14323.5575, "last_run_timestamp": "2026-10-31T12:30:14.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.764, "accuracy_std_dev": 0.020483142122242875, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.080228768, "avg_duration_per_task_ms": 494377.212, "avg_output_tokens_per_task": 14669.972, "last_run_timestamp": "2026-10-31T12:27:34.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7639395454545455, "accuracy_std_dev": 0.035811554789337924, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.19443994909090911, "avg_duration_per_task_ms": 474708.6345454545, "avg_output_tokens_per_task": 27695.465454545454, "last_run_timestamp": "2026-10-31T19:55:50.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7637037777777778, "accuracy_std_dev": 0.021798347556015337, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06269429111111112, "avg_duration_per_task_ms": 113855.65555555555, "avg_output_tokens_per_task": 4583.311111111111, "last_run_timestamp": "2026-10-31T12:40:08.732Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7637036666666667, "accuracy_std_dev": 0.01835498758739252, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029433213333333333, "avg_duration_per_task_ms": 122522.84, "avg_output_tokens_per_task": 12030.735555555555, "last_run_timestamp": "2026-10-31T19:08:17.847Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7633335, "accuracy_std_dev": 0.06984115707983368, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.251395065, "avg_duration_per_task_ms": 485523.98, "avg_output_tokens_per_task": 29105.6325, "last_run_timestamp": "2026-10-31T19:26:18.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.763333375, "accuracy_std_dev": 0.024267095921934618, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0630558525, "avg_duration_per_task_ms": 1180279.39, "avg_output_tokens_per_task": 60139.9825, "last_run_timestamp": "2026-10-31T19:16:07.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7633333333333333, "accuracy_std_dev": 0.03134832186896135, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014250108888888892, "avg_duration_per_task_ms": 231681.52, "avg_output_tokens_per_task": 15957.917777777779, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01189864, "avg_duration_per_task_ms": 642869.1, "avg_output_tokens_per_task": 23140.98, "last_run_timestamp": "2026-10-24T13:28:08.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "CoreWeave", "accuracy": 0.763333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025058120000000003, "avg_duration_per_task_ms": 106870.58, "avg_output_tokens_per_task": 12582.44, "last_run_timestamp": "2026-11-04T18:35:21.465Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020903190000000002, "avg_duration_per_task_ms": 62428.04, "avg_output_tokens_per_task": 8093.56, "last_run_timestamp": "2026-10-31T12:43:49.743Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.763333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05078266, "avg_duration_per_task_ms": 205280.74, "avg_output_tokens_per_task": 7724.94, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7630303636363637, "accuracy_std_dev": 0.027171604447132024, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5365658963636364, "avg_duration_per_task_ms": 173544.41818181818, "avg_output_tokens_per_task": 10675.270909090908, "last_run_timestamp": "2026-10-31T11:55:38.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7630302727272728, "accuracy_std_dev": 0.03170123269496259, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.15556449636363637, "avg_duration_per_task_ms": 593095.6363636364, "avg_output_tokens_per_task": 12420.629090909091, "last_run_timestamp": "2026-10-31T13:02:37.599Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7629628888888889, "accuracy_std_dev": 0.03837586761797183, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.00882314, "avg_duration_per_task_ms": 182772.50444444444, "avg_output_tokens_per_task": 15534.135555555555, "last_run_timestamp": "2026-10-31T19:07:06.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7629628888888889, "accuracy_std_dev": 0.03191221842089971, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02681685777777778, "avg_duration_per_task_ms": 156543.7, "avg_output_tokens_per_task": 9000.691111111111, "last_run_timestamp": "2026-10-24T19:16:17.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7627778333333333, "accuracy_std_dev": 0.03313374040368652, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07174823666666666, "avg_duration_per_task_ms": 893696.3066666666, "avg_output_tokens_per_task": 51834.78666666667, "last_run_timestamp": "2026-10-27T03:58:39.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7626668, "accuracy_std_dev": 0.014966487007978884, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009859876, "avg_duration_per_task_ms": 578544.56, "avg_output_tokens_per_task": 20173.776, "last_run_timestamp": "2026-10-31T14:48:08.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7626666, "accuracy_std_dev": 0.02954472165785287, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009234846, "avg_duration_per_task_ms": 195074.734, "avg_output_tokens_per_task": 4475.264, "last_run_timestamp": "2026-10-31T12:25:34.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7625418160535117, "accuracy_std_dev": 0.02147753020972668, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.09006360702341139, "avg_duration_per_task_ms": 48936.09866220736, "avg_output_tokens_per_task": 4348.224080267559, "last_run_timestamp": "2026-10-31T12:39:14.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.7625417625418061, "accuracy_std_dev": 0.01910679234955175, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.03341546153846154, "avg_duration_per_task_ms": 144903.0602006689, "avg_output_tokens_per_task": 21777.678929765887, "last_run_timestamp": "2026-10-31T12:18:15.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.7625, "accuracy_std_dev": 0.028467730270430754, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0217337575, "avg_duration_per_task_ms": 233098.835, "avg_output_tokens_per_task": 12085.595, "last_run_timestamp": "2026-10-31T19:23:14.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7623932051282051, "accuracy_std_dev": 0.018251804130003154, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.24276578461538462, "avg_duration_per_task_ms": 125754.82051282052, "avg_output_tokens_per_task": 7748.235897435898, "last_run_timestamp": "2026-10-31T11:39:17.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7623223984962407, "accuracy_std_dev": 0.0190669296226313, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.011861127819548874, "avg_duration_per_task_ms": 357840.4411027569, "avg_output_tokens_per_task": 17112.51127819549, "last_run_timestamp": "2026-10-31T11:47:32.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7622220000000001, "accuracy_std_dev": 0.008314764498569207, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.007347476666666667, "avg_duration_per_task_ms": 300690.79, "avg_output_tokens_per_task": 12348.97, "last_run_timestamp": "2026-10-31T20:10:56.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.762177653295129, "accuracy_std_dev": 0.016008036097981727, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.027902553008595985, "avg_duration_per_task_ms": 104077.65616045846, "avg_output_tokens_per_task": 17491.627507163324, "last_run_timestamp": "2026-10-31T11:55:25.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7621775042979944, "accuracy_std_dev": 0.026194934880387766, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.06229761318051576, "avg_duration_per_task_ms": 372098.9226361032, "avg_output_tokens_per_task": 49911.94555873926, "last_run_timestamp": "2026-10-31T19:59:18.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.7619782998454405, "accuracy_std_dev": 0.039147489254698645, "run_count": 13, "total_questions": 647, "avg_cost_per_task_usd": 0.06417079598145287, "avg_duration_per_task_ms": 203709.1777434312, "avg_output_tokens_per_task": 11216.542503863988, "last_run_timestamp": "2026-10-31T14:18:08.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.761904902255639, "accuracy_std_dev": 0.022085869925347657, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.010246486215538846, "avg_duration_per_task_ms": 307475.68922305765, "avg_output_tokens_per_task": 12562.348370927319, "last_run_timestamp": "2026-10-31T19:48:48.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.04042854562846226, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11324955142857143, "avg_duration_per_task_ms": 271951.16285714286, "avg_output_tokens_per_task": 11844.885714285714, "last_run_timestamp": "2026-10-31T21:36:17.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.7617847878787878, "accuracy_std_dev": 0.03876959831491683, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.02413183333333333, "avg_duration_per_task_ms": 113963.81313131313, "avg_output_tokens_per_task": 15997.81313131313, "last_run_timestamp": "2026-10-24T12:54:33.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.761784404040404, "accuracy_std_dev": 0.046884570628299496, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.03727159090909091, "avg_duration_per_task_ms": 138498.65151515152, "avg_output_tokens_per_task": 4381.035353535353, "last_run_timestamp": "2026-10-31T12:40:46.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.76166675, "accuracy_std_dev": 0.025404142635027726, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02248819, "avg_duration_per_task_ms": 170461.53666666665, "avg_output_tokens_per_task": 16469.198333333334, "last_run_timestamp": "2026-10-31T12:11:52.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7614249072580644, "accuracy_std_dev": 0.03191360961423825, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.04908138104838709, "avg_duration_per_task_ms": 385516.27419354836, "avg_output_tokens_per_task": 12910.963709677419, "last_run_timestamp": "2026-10-31T13:30:45.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7612120909090909, "accuracy_std_dev": 0.02288640210595304, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.54388782, "avg_duration_per_task_ms": 180374.2618181818, "avg_output_tokens_per_task": 10717.156363636364, "last_run_timestamp": "2026-10-31T12:58:32.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7612120909090909, "accuracy_std_dev": 0.01903078652391983, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03308226727272728, "avg_duration_per_task_ms": 220782.66727272727, "avg_output_tokens_per_task": 15395.681818181818, "last_run_timestamp": "2026-10-31T12:46:55.135Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7611108333333334, "accuracy_std_dev": 0.02586761405320473, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.2186762766666667, "avg_duration_per_task_ms": 724933.3133333334, "avg_output_tokens_per_task": 57844.28333333333, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7609786762860727, "accuracy_std_dev": 0.02511202768812936, "run_count": 16, "total_questions": 797, "avg_cost_per_task_usd": 0.054708953575909665, "avg_duration_per_task_ms": 274645.58720200753, "avg_output_tokens_per_task": 12041.819322459221, "last_run_timestamp": "2026-10-31T12:07:50.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.01649574672206044, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08085598000000001, "avg_duration_per_task_ms": 105369.85714285714, "avg_output_tokens_per_task": 22435.354285714286, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7607886785714285, "accuracy_std_dev": 0.017367969160654122, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.018826165178571424, "avg_duration_per_task_ms": 120325.66071428571, "avg_output_tokens_per_task": 14433.707589285714, "last_run_timestamp": "2026-10-31T12:07:52.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.034633054297881376, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.006436806666666667, "avg_duration_per_task_ms": 125547.04444444444, "avg_output_tokens_per_task": 4262.8533333333335, "last_run_timestamp": "2026-10-24T14:25:23.734Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7606060909090909, "accuracy_std_dev": 0.02369062226226981, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03764717818181818, "avg_duration_per_task_ms": 169270.9309090909, "avg_output_tokens_per_task": 16441.54909090909, "last_run_timestamp": "2026-10-31T12:45:56.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.7604166354166666, "accuracy_std_dev": 0.010476531320050537, "run_count": 4, "total_questions": 192, "avg_cost_per_task_usd": 0.29691326041666666, "avg_duration_per_task_ms": 310718.984375, "avg_output_tokens_per_task": 7030.182291666667, "last_run_timestamp": "2026-10-31T12:13:14.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7603703333333334, "accuracy_std_dev": 0.033126907327897526, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08524436222222224, "avg_duration_per_task_ms": 566210.18, "avg_output_tokens_per_task": 12520.904444444444, "last_run_timestamp": "2026-10-31T12:26:16.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.7601351621621621, "accuracy_std_dev": 0.012280818789432917, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.032259290540540546, "avg_duration_per_task_ms": 475460.4324324324, "avg_output_tokens_per_task": 12381.702702702703, "last_run_timestamp": "2026-10-31T12:56:10.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.7600002499999999, "accuracy_std_dev": 0.008164965810553043, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015787874999999996, "avg_duration_per_task_ms": 119015.24, "avg_output_tokens_per_task": 12260.915, "last_run_timestamp": "2026-10-31T13:06:19.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.7600002499999999, "accuracy_std_dev": 0.027080189561513434, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02989183, "avg_duration_per_task_ms": 125270.795, "avg_output_tokens_per_task": 12269.195, "last_run_timestamp": "2026-10-31T12:01:23.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.7600001999999999, "accuracy_std_dev": 0.03126929626582598, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.114411384, "avg_duration_per_task_ms": 874872.748, "avg_output_tokens_per_task": 24203.156, "last_run_timestamp": "2026-10-31T12:44:40.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.76, "accuracy_std_dev": 0.03311047725083071, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01778674, "avg_duration_per_task_ms": 293727.4111111111, "avg_output_tokens_per_task": 11448.377777777778, "last_run_timestamp": "2026-10-31T20:10:47.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Pearl Research", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11851576, "avg_duration_per_task_ms": 319323.58, "avg_output_tokens_per_task": 19182.22, "last_run_timestamp": "2026-11-04T15:08:37.819Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019761673333333334, "avg_duration_per_task_ms": 291193.25333333336, "avg_output_tokens_per_task": 17811.393333333333, "last_run_timestamp": "2026-10-10T11:23:23.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.3443486, "avg_duration_per_task_ms": 184596.7, "avg_output_tokens_per_task": 12502.76, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Morph", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27690942, "avg_duration_per_task_ms": 230447.6, "avg_output_tokens_per_task": 10636.8, "last_run_timestamp": "2026-11-05T06:25:26.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.76, "accuracy_std_dev": 0.005443582705045132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.40005456666666667, "avg_duration_per_task_ms": 394539.76666666666, "avg_output_tokens_per_task": 34690.113333333335, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.025991443038046205, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08891490399999999, "avg_duration_per_task_ms": 301406.882, "avg_output_tokens_per_task": 12044.934, "last_run_timestamp": "2026-10-31T12:46:14.833Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10151575999999998, "avg_duration_per_task_ms": 434936.03, "avg_output_tokens_per_task": 13962.46, "last_run_timestamp": "2026-10-24T12:32:07.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.40321123999999997, "avg_duration_per_task_ms": 34129.92, "avg_output_tokens_per_task": 9245.3, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7599999000000001, "accuracy_std_dev": 0.030840204640858015, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008153706000000002, "avg_duration_per_task_ms": 200971.108, "avg_output_tokens_per_task": 4219.412, "last_run_timestamp": "2026-10-31T12:49:39.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.759999875, "accuracy_std_dev": 0.030368121078844755, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10768833249999998, "avg_duration_per_task_ms": 237288.72, "avg_output_tokens_per_task": 11168.8675, "last_run_timestamp": "2026-10-31T20:03:31.357Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.759846288184438, "accuracy_std_dev": 0.015452349289717374, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.15869831700288184, "avg_duration_per_task_ms": 791662.6916426513, "avg_output_tokens_per_task": 30669.08069164265, "last_run_timestamp": "2026-10-31T19:07:30.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7594444166666666, "accuracy_std_dev": 0.02168086358388558, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.021428943333333332, "avg_duration_per_task_ms": 120315.54166666667, "avg_output_tokens_per_task": 17324.703333333335, "last_run_timestamp": "2026-10-31T12:41:20.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7594444166666666, "accuracy_std_dev": 0.033715523315080306, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09267692833333334, "avg_duration_per_task_ms": 244983.77333333335, "avg_output_tokens_per_task": 12483.865, "last_run_timestamp": "2026-10-31T12:53:55.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7593939090909091, "accuracy_std_dev": 0.01916536035974823, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.01099011090909091, "avg_duration_per_task_ms": 321334.9527272727, "avg_output_tokens_per_task": 14815.935454545455, "last_run_timestamp": "2026-11-06T22:45:20.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.7592155882352941, "accuracy_std_dev": 0.027618637596970693, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.011689743529411765, "avg_duration_per_task_ms": 166650.9011764706, "avg_output_tokens_per_task": 5474.670588235294, "last_run_timestamp": "2026-10-31T12:46:12.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.759157608058608, "accuracy_std_dev": 0.022493444068575088, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.01790187728937729, "avg_duration_per_task_ms": 191122.336996337, "avg_output_tokens_per_task": 6046.5641025641025, "last_run_timestamp": "2026-11-01T11:24:36.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "InferenceNet", "accuracy": 0.7590360803212852, "accuracy_std_dev": 0.01520064937560236, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.21346128514056226, "avg_duration_per_task_ms": 312634.5903614458, "avg_output_tokens_per_task": 10222.072289156626, "last_run_timestamp": "2026-11-05T00:24:27.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7587878181818182, "accuracy_std_dev": 0.01653114932933463, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.17482494545454544, "avg_duration_per_task_ms": 315756.4018181818, "avg_output_tokens_per_task": 10740.534545454546, "last_run_timestamp": "2026-10-31T11:13:27.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7585184444444445, "accuracy_std_dev": 0.023680696011313677, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.010843522222222221, "avg_duration_per_task_ms": 1028208.5311111112, "avg_output_tokens_per_task": 13132.244444444445, "last_run_timestamp": "2026-10-31T20:29:45.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7583333333333333, "accuracy_std_dev": 0.027772194995874723, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5207035750000001, "avg_duration_per_task_ms": 214690.15666666668, "avg_output_tokens_per_task": 9895.341666666667, "last_run_timestamp": "2026-10-31T12:37:18.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.7583333333333333, "accuracy_std_dev": 0.03125451671181126, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06620967333333333, "avg_duration_per_task_ms": 161431.49, "avg_output_tokens_per_task": 10316.483333333334, "last_run_timestamp": "2026-10-31T19:43:05.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.014433785597946933, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.42515429, "avg_duration_per_task_ms": 460450.54, "avg_output_tokens_per_task": 34342.89, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.7580951428571429, "accuracy_std_dev": 0.016605241912709113, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.048255082857142854, "avg_duration_per_task_ms": 69808.27142857143, "avg_output_tokens_per_task": 4328.277142857143, "last_run_timestamp": "2026-10-31T12:44:24.094Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7578788636363636, "accuracy_std_dev": 0.03116509593832564, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5560903545454545, "avg_duration_per_task_ms": 181840.00545454546, "avg_output_tokens_per_task": 10618.228181818182, "last_run_timestamp": "2026-10-31T12:37:18.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7577777777777778, "accuracy_std_dev": 0.026480824650543643, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.025315553333333334, "avg_duration_per_task_ms": 238703.38, "avg_output_tokens_per_task": 12990.084444444445, "last_run_timestamp": "2026-10-31T19:02:15.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7577777777777778, "accuracy_std_dev": 0.01859267080077988, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1710706822222222, "avg_duration_per_task_ms": 189011.57111111112, "avg_output_tokens_per_task": 11157.495555555555, "last_run_timestamp": "2026-10-31T19:18:33.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.28612407999999995, "avg_duration_per_task_ms": 2065607.3133333332, "avg_output_tokens_per_task": 10347.066666666668, "last_run_timestamp": "2026-10-31T11:54:15.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.040123113009281265, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.15586289333333334, "avg_duration_per_task_ms": 442954.5133333333, "avg_output_tokens_per_task": 26206.566666666666, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09565631333333334, "avg_duration_per_task_ms": 177852.71333333335, "avg_output_tokens_per_task": 9687.686666666666, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.7575, "accuracy_std_dev": 0.02184208476199102, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0299009825, "avg_duration_per_task_ms": 207569.0425, "avg_output_tokens_per_task": 13839.6975, "last_run_timestamp": "2026-10-31T19:55:08.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.7575, "accuracy_std_dev": 0.05171382256708548, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.26214604, "avg_duration_per_task_ms": 26048.875, "avg_output_tokens_per_task": 12109.95, "last_run_timestamp": "2026-10-31T11:18:07.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7575, "accuracy_std_dev": 0.023078766095699303, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012481125, "avg_duration_per_task_ms": 215467.87, "avg_output_tokens_per_task": 4890.4975, "last_run_timestamp": "2026-10-31T11:56:22.568Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7571111333333334, "accuracy_std_dev": 0.02820195562927429, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.09072873866666667, "avg_duration_per_task_ms": 188985.284, "avg_output_tokens_per_task": 11040.473333333333, "last_run_timestamp": "2026-10-31T19:54:53.711Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.757037111111111, "accuracy_std_dev": 0.029162999653078087, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04487014222222222, "avg_duration_per_task_ms": 185453.6288888889, "avg_output_tokens_per_task": 9571.857777777777, "last_run_timestamp": "2026-10-31T20:17:26.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7569698181818182, "accuracy_std_dev": 0.01591999626437932, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07314548, "avg_duration_per_task_ms": 513687.81272727274, "avg_output_tokens_per_task": 32116.561818181817, "last_run_timestamp": "2026-10-31T20:08:33.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7568888666666667, "accuracy_std_dev": 0.02909656905746029, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.008429334666666666, "avg_duration_per_task_ms": 120051.87733333334, "avg_output_tokens_per_task": 4480.442666666667, "last_run_timestamp": "2026-11-04T21:55:13.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7568491678082192, "accuracy_std_dev": 0.014345100754845415, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.03330520547945206, "avg_duration_per_task_ms": 2320790.7157534244, "avg_output_tokens_per_task": 51570.45205479452, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.756708080495356, "accuracy_std_dev": 0.027120750325406145, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.19143365634674925, "avg_duration_per_task_ms": 498299.07585139317, "avg_output_tokens_per_task": 20833.128482972137, "last_run_timestamp": "2026-10-31T14:12:32.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08839812, "avg_duration_per_task_ms": 73380.11, "avg_output_tokens_per_task": 8719.83, "last_run_timestamp": "2026-10-31T12:44:24.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Wafer", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17797051000000003, "avg_duration_per_task_ms": 171587.38, "avg_output_tokens_per_task": 10362.94, "last_run_timestamp": "2026-11-05T21:56:52.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06446149, "avg_duration_per_task_ms": 318929.97, "avg_output_tokens_per_task": 13284.66, "last_run_timestamp": "2026-10-31T12:25:36.318Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7566668, "accuracy_std_dev": 0.01819650760338369, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.151380274, "avg_duration_per_task_ms": 165330.14, "avg_output_tokens_per_task": 19976.098, "last_run_timestamp": "2026-10-31T12:20:13.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.022026835486641215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06927284666666667, "avg_duration_per_task_ms": 313911.26, "avg_output_tokens_per_task": 14319.526666666667, "last_run_timestamp": "2026-10-31T12:43:48.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7564444666666666, "accuracy_std_dev": 0.023958871344776582, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.04415017200000001, "avg_duration_per_task_ms": 138405.18933333334, "avg_output_tokens_per_task": 4510.322666666667, "last_run_timestamp": "2026-10-31T12:43:47.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.7563636363636363, "accuracy_std_dev": 0.033917706275022164, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.019490098181818183, "avg_duration_per_task_ms": 208148.6509090909, "avg_output_tokens_per_task": 15196.68909090909, "last_run_timestamp": "2026-10-31T12:48:59.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7562961111111112, "accuracy_std_dev": 0.02966657881426859, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03134207111111111, "avg_duration_per_task_ms": 175492.92222222223, "avg_output_tokens_per_task": 14182.893333333333, "last_run_timestamp": "2026-10-31T11:54:37.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7561521722595078, "accuracy_std_dev": 0.03542861834034983, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.10875903355704698, "avg_duration_per_task_ms": 155583.68456375838, "avg_output_tokens_per_task": 9207.604026845638, "last_run_timestamp": "2026-10-31T19:46:27.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7559789775784755, "accuracy_std_dev": 0.038608976715106755, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.090114966367713, "avg_duration_per_task_ms": 617883.6928251121, "avg_output_tokens_per_task": 10068.533632286995, "last_run_timestamp": "2026-10-31T20:25:10.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7558333333333334, "accuracy_std_dev": 0.021131098726495254, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.01632131833333333, "avg_duration_per_task_ms": 115260.22166666666, "avg_output_tokens_per_task": 4430.61, "last_run_timestamp": "2026-10-31T12:09:39.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.75583325, "accuracy_std_dev": 0.028855413030963544, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012369372500000001, "avg_duration_per_task_ms": 223619.505, "avg_output_tokens_per_task": 13737.5625, "last_run_timestamp": "2026-10-31T20:16:35.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7557575454545454, "accuracy_std_dev": 0.021134116315326572, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0626866909090909, "avg_duration_per_task_ms": 166564.66363636364, "avg_output_tokens_per_task": 4635.490909090909, "last_run_timestamp": "2026-10-31T13:36:23.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7557471896551724, "accuracy_std_dev": 0.051344765491603474, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07958818965517242, "avg_duration_per_task_ms": 709509.3793103448, "avg_output_tokens_per_task": 47576.2183908046, "last_run_timestamp": "2026-10-31T18:44:20.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.7555555, "accuracy_std_dev": 0.015947487129638963, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.0058668516666666665, "avg_duration_per_task_ms": 265133.9683333333, "avg_output_tokens_per_task": 15836.96, "last_run_timestamp": "2026-10-31T13:22:04.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.7555554444444444, "accuracy_std_dev": 0.018856167739030906, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0086613, "avg_duration_per_task_ms": 117282.64, "avg_output_tokens_per_task": 5380.108888888889, "last_run_timestamp": "2026-10-31T13:20:27.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7554545454545455, "accuracy_std_dev": 0.03830433230509108, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.16801238363636364, "avg_duration_per_task_ms": 309650.71636363637, "avg_output_tokens_per_task": 24145.685454545455, "last_run_timestamp": "2026-10-31T13:11:05.620Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7553333, "accuracy_std_dev": 0.03334003466119973, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04678315799999999, "avg_duration_per_task_ms": 241838.904, "avg_output_tokens_per_task": 14405.774, "last_run_timestamp": "2026-10-31T12:45:59.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7552378571428572, "accuracy_std_dev": 0.014126057041900291, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009880265714285713, "avg_duration_per_task_ms": 245506.26571428572, "avg_output_tokens_per_task": 16074.99142857143, "last_run_timestamp": "2026-10-31T12:27:44.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7552085625, "accuracy_std_dev": 0.029782500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.02997340625, "avg_duration_per_task_ms": 1679708.6145833333, "avg_output_tokens_per_task": 72130.22916666667, "last_run_timestamp": "2026-10-31T20:01:41.151Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.7551024285714285, "accuracy_std_dev": 0.011805499999999969, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.27501580612244897, "avg_duration_per_task_ms": 65997.72448979592, "avg_output_tokens_per_task": 8842.591836734693, "last_run_timestamp": "2026-10-31T12:36:21.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.7550335167785235, "accuracy_std_dev": 0.023364430687122125, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.012232758389261746, "avg_duration_per_task_ms": 267603.8926174497, "avg_output_tokens_per_task": 4166.208053691275, "last_run_timestamp": "2026-10-31T12:14:04.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7549998, "accuracy_std_dev": 0.029636688974310207, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014858368, "avg_duration_per_task_ms": 71120.474, "avg_output_tokens_per_task": 4319.272, "last_run_timestamp": "2026-10-31T11:35:00.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7547041935483871, "accuracy_std_dev": 0.023888446777469652, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.01950188306451613, "avg_duration_per_task_ms": 216554.71370967742, "avg_output_tokens_per_task": 4241.487903225807, "last_run_timestamp": "2026-10-31T11:51:07.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.75416675, "accuracy_std_dev": 0.019096569464219906, "run_count": 13, "total_questions": 640, "avg_cost_per_task_usd": 0.0324113640625, "avg_duration_per_task_ms": 163294.796875, "avg_output_tokens_per_task": 15840.8265625, "last_run_timestamp": "2026-10-31T11:41:35.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7541666875, "accuracy_std_dev": 0.02875898591575064, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.20787477937499993, "avg_duration_per_task_ms": 114719.823125, "avg_output_tokens_per_task": 7707.7675, "last_run_timestamp": "2026-10-31T12:48:39.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7540880930817611, "accuracy_std_dev": 0.02434701736834258, "run_count": 16, "total_questions": 795, "avg_cost_per_task_usd": 0.08681218238993711, "avg_duration_per_task_ms": 616702.9220125786, "avg_output_tokens_per_task": 21133.411320754716, "last_run_timestamp": "2026-11-06T22:40:08.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.7539393636363636, "accuracy_std_dev": 0.030411866372049668, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5373833490909091, "avg_duration_per_task_ms": 189228.24181818182, "avg_output_tokens_per_task": 10661.903636363637, "last_run_timestamp": "2026-10-31T12:48:37.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7538888333333333, "accuracy_std_dev": 0.023007058767667125, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.007775635, "avg_duration_per_task_ms": 226184.26333333334, "avg_output_tokens_per_task": 14761.815, "last_run_timestamp": "2026-10-31T13:30:46.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16966541999999996, "avg_duration_per_task_ms": 147439, "avg_output_tokens_per_task": 22903.86, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07113415, "avg_duration_per_task_ms": 677434.95, "avg_output_tokens_per_task": 13100.77, "last_run_timestamp": "2026-10-31T13:06:57.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.01721340993398913, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.33945626, "avg_duration_per_task_ms": 116389.55333333333, "avg_output_tokens_per_task": 7044.336666666667, "last_run_timestamp": "2026-10-31T12:36:01.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.03925236487426234, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03720142666666666, "avg_duration_per_task_ms": 244561.18666666668, "avg_output_tokens_per_task": 13729.346666666666, "last_run_timestamp": "2026-10-31T19:28:27.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02277092037860767, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020851383333333334, "avg_duration_per_task_ms": 133350.03333333333, "avg_output_tokens_per_task": 13555.86, "last_run_timestamp": "2026-10-31T19:21:19.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018967786666666667, "avg_duration_per_task_ms": 85971.64666666667, "avg_output_tokens_per_task": 11730.173333333334, "last_run_timestamp": "2026-10-31T12:40:46.055Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.019625871060640607, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06505746, "avg_duration_per_task_ms": 250536.72666666665, "avg_output_tokens_per_task": 8651.606666666667, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.029186609056590034, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09943221, "avg_duration_per_task_ms": 284246.225, "avg_output_tokens_per_task": 12177.723333333333, "last_run_timestamp": "2026-10-31T12:41:58.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02000009259423011, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.009942297777777777, "avg_duration_per_task_ms": 208204.5977777778, "avg_output_tokens_per_task": 4241.026666666667, "last_run_timestamp": "2026-11-06T22:25:00.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7533332857142857, "accuracy_std_dev": 0.015118704911601446, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3460215285714286, "avg_duration_per_task_ms": 119668.60857142857, "avg_output_tokens_per_task": 7072.9971428571425, "last_run_timestamp": "2026-10-31T12:24:55.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7533332727272728, "accuracy_std_dev": 0.037172733573000175, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.044545774545454546, "avg_duration_per_task_ms": 178434.63818181818, "avg_output_tokens_per_task": 19162.814545454545, "last_run_timestamp": "2026-10-31T12:26:48.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7533332500000001, "accuracy_std_dev": 0.030731887168501403, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0298026275, "avg_duration_per_task_ms": 427042.9575, "avg_output_tokens_per_task": 11770.6425, "last_run_timestamp": "2026-10-31T19:35:25.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01197713, "avg_duration_per_task_ms": 212560.45, "avg_output_tokens_per_task": 13957.42, "last_run_timestamp": "2026-10-24T11:49:49.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.7-20260916", "model_slug": "x-ai/grok-4.7", "model_name": "SpaceXAI: Grok 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29315726000000003, "avg_duration_per_task_ms": 206111.18, "avg_output_tokens_per_task": 11166.84, "last_run_timestamp": "2026-11-03T04:31:24.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7528342925170068, "accuracy_std_dev": 0.024943599437308363, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.1277965918367347, "avg_duration_per_task_ms": 243630.26530612246, "avg_output_tokens_per_task": 45142.87074829932, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.752715030075188, "accuracy_std_dev": 0.009940333684036977, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.02512772681704261, "avg_duration_per_task_ms": 75665.01253132832, "avg_output_tokens_per_task": 12251.558897243107, "last_run_timestamp": "2026-10-31T12:26:39.122Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7526667, "accuracy_std_dev": 0.02374861961062159, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.007248376000000002, "avg_duration_per_task_ms": 194889.812, "avg_output_tokens_per_task": 4384.802, "last_run_timestamp": "2026-10-31T13:46:40.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7525287586206897, "accuracy_std_dev": 0.020184097544858358, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.2696313268965517, "avg_duration_per_task_ms": 233088.7579310345, "avg_output_tokens_per_task": 11456.733793103449, "last_run_timestamp": "2026-11-04T21:30:17.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7525, "accuracy_std_dev": 0.022715180925979885, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02207197, "avg_duration_per_task_ms": 477011.815, "avg_output_tokens_per_task": 13991.755, "last_run_timestamp": "2026-10-31T19:04:16.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.752499875, "accuracy_std_dev": 0.028173595055643437, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.18478309000000004, "avg_duration_per_task_ms": 179248.5325, "avg_output_tokens_per_task": 9658.775, "last_run_timestamp": "2026-10-31T20:09:19.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7523809047619048, "accuracy_std_dev": 0.017612644913472624, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.36889707904761904, "avg_duration_per_task_ms": 141180.9876190476, "avg_output_tokens_per_task": 7127.063809523809, "last_run_timestamp": "2026-10-31T12:26:17.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7523076923076923, "accuracy_std_dev": 0.0332661532107905, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.10102499846153845, "avg_duration_per_task_ms": 302464.04, "avg_output_tokens_per_task": 18714.755384615386, "last_run_timestamp": "2026-10-31T13:11:23.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.752222375, "accuracy_std_dev": 0.02258048225941397, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.24382183083333336, "avg_duration_per_task_ms": 219317.55666666667, "avg_output_tokens_per_task": 10147.890833333333, "last_run_timestamp": "2026-10-31T12:43:33.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7522221666666666, "accuracy_std_dev": 0.016960131174971006, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.12217692666666667, "avg_duration_per_task_ms": 121673.44, "avg_output_tokens_per_task": 10724.803333333333, "last_run_timestamp": "2026-10-17T19:14:08.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7519260539291217, "accuracy_std_dev": 0.019651146282719818, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.027055047765793528, "avg_duration_per_task_ms": 539313.875192604, "avg_output_tokens_per_task": 22031.878274268103, "last_run_timestamp": "2026-10-31T19:23:15.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.7516778724832214, "accuracy_std_dev": 0.030780262062995275, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.08526638255033557, "avg_duration_per_task_ms": 172059.79865771811, "avg_output_tokens_per_task": 8282.208053691274, "last_run_timestamp": "2026-10-31T11:50:00.858Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7514287142857142, "accuracy_std_dev": 0.02844421520758676, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.032982819999999996, "avg_duration_per_task_ms": 221535.9714285714, "avg_output_tokens_per_task": 35206.688571428575, "last_run_timestamp": "2026-10-31T11:46:18.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7513043478260869, "accuracy_std_dev": 0.029647460490422974, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.20092619999999997, "avg_duration_per_task_ms": 96197.9252173913, "avg_output_tokens_per_task": 7585.031304347826, "last_run_timestamp": "2026-10-31T12:38:13.068Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.751256015075377, "accuracy_std_dev": 0.006852721115549665, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.07795362814070352, "avg_duration_per_task_ms": 228603.50753768845, "avg_output_tokens_per_task": 10368.422110552763, "last_run_timestamp": "2026-10-10T19:17:08.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.751111, "accuracy_std_dev": 0.022662199996175744, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05099817333333333, "avg_duration_per_task_ms": 373986.0133333333, "avg_output_tokens_per_task": 8579.546666666667, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7509091818181818, "accuracy_std_dev": 0.030784008944486355, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.20063707090909091, "avg_duration_per_task_ms": 117571.03454545455, "avg_output_tokens_per_task": 7764.232727272727, "last_run_timestamp": "2026-10-31T11:54:55.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.750909, "accuracy_std_dev": 0.015640538557339927, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03659924545454545, "avg_duration_per_task_ms": 266725.77636363637, "avg_output_tokens_per_task": 21362.425454545453, "last_run_timestamp": "2026-10-31T20:23:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.750833375, "accuracy_std_dev": 0.029237916546573147, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.025213935, "avg_duration_per_task_ms": 300217.2775, "avg_output_tokens_per_task": 10899.815, "last_run_timestamp": "2026-10-31T19:15:14.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.7506668, "accuracy_std_dev": 0.025854254755455656, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.264866352, "avg_duration_per_task_ms": 264537.464, "avg_output_tokens_per_task": 10440.212, "last_run_timestamp": "2026-10-24T11:49:28.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7506666, "accuracy_std_dev": 0.03785357285435553, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007070539999999999, "avg_duration_per_task_ms": 292989.264, "avg_output_tokens_per_task": 3954.392, "last_run_timestamp": "2026-10-31T13:04:37.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7504761428571428, "accuracy_std_dev": 0.04779025223509368, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.020752462857142853, "avg_duration_per_task_ms": 333009.8742857143, "avg_output_tokens_per_task": 11171.528571428571, "last_run_timestamp": "2026-10-31T13:00:18.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7502563846153846, "accuracy_std_dev": 0.02447608447990521, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.09372697076923077, "avg_duration_per_task_ms": 135890.0169230769, "avg_output_tokens_per_task": 12287.435384615384, "last_run_timestamp": "2026-10-31T18:56:42.115Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.750000125, "accuracy_std_dev": 0.019720301160463437, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.013962239999999999, "avg_duration_per_task_ms": 241952.985, "avg_output_tokens_per_task": 14693.71, "last_run_timestamp": "2026-10-31T19:19:01.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29130333999999997, "avg_duration_per_task_ms": 176962.37, "avg_output_tokens_per_task": 10386.26, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.75, "accuracy_std_dev": 0.028631403807707387, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04503071555555555, "avg_duration_per_task_ms": 450907.20666666667, "avg_output_tokens_per_task": 22758.244444444445, "last_run_timestamp": "2026-10-24T19:38:23.863Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7499999999999999, "accuracy_std_dev": 0.03349968490896593, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028200379999999997, "avg_duration_per_task_ms": 163955.21, "avg_output_tokens_per_task": 15323.823333333334, "last_run_timestamp": "2026-10-31T12:28:07.595Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7498989696969697, "accuracy_std_dev": 0.03411693737496916, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.20618370909090916, "avg_duration_per_task_ms": 91953.36, "avg_output_tokens_per_task": 7458.127878787879, "last_run_timestamp": "2026-10-31T13:20:20.506Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7496970909090909, "accuracy_std_dev": 0.10488520113867744, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1517481509090909, "avg_duration_per_task_ms": 559680.3290909091, "avg_output_tokens_per_task": 28098.66, "last_run_timestamp": "2026-10-31T19:37:30.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.749524, "accuracy_std_dev": 0.021633867377662402, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.35241304, "avg_duration_per_task_ms": 87048.52428571429, "avg_output_tokens_per_task": 6993.714285714285, "last_run_timestamp": "2026-10-31T11:29:30.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.7495238571428572, "accuracy_std_dev": 0.021338220105372136, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.016605834285714285, "avg_duration_per_task_ms": 72972.84285714285, "avg_output_tokens_per_task": 4253.202857142857, "last_run_timestamp": "2026-10-31T12:03:51.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7494446666666666, "accuracy_std_dev": 0.029496932253861397, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.34673691166666676, "avg_duration_per_task_ms": 225396.725, "avg_output_tokens_per_task": 10875.54, "last_run_timestamp": "2026-10-31T12:24:26.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7493333, "accuracy_std_dev": 0.015549210771289982, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.012228522, "avg_duration_per_task_ms": 315949.068, "avg_output_tokens_per_task": 16300.002, "last_run_timestamp": "2026-10-31T11:27:38.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7492774046242774, "accuracy_std_dev": 0.025230016427736986, "run_count": 14, "total_questions": 692, "avg_cost_per_task_usd": 0.013838482658959535, "avg_duration_per_task_ms": 56300.32225433526, "avg_output_tokens_per_task": 4085.725433526012, "last_run_timestamp": "2026-10-31T12:14:05.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.749091, "accuracy_std_dev": 0.0224325111713902, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1957560618181818, "avg_duration_per_task_ms": 114755.80545454545, "avg_output_tokens_per_task": 7618.518181818182, "last_run_timestamp": "2026-10-31T11:08:14.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7490679239373601, "accuracy_std_dev": 0.028614120182218948, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.09606371140939597, "avg_duration_per_task_ms": 393542.0715883669, "avg_output_tokens_per_task": 24831.977628635348, "last_run_timestamp": "2026-10-31T12:39:48.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.748889, "accuracy_std_dev": 0.01911599421078245, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019773013333333336, "avg_duration_per_task_ms": 225130.51333333334, "avg_output_tokens_per_task": 12421.726666666667, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.748889, "accuracy_std_dev": 0.015713326891527487, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03620722, "avg_duration_per_task_ms": 114735.25333333333, "avg_output_tokens_per_task": 14087.753333333334, "last_run_timestamp": "2026-10-31T11:35:01.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7488888888888889, "accuracy_std_dev": 0.029814247982557492, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.012581893333333335, "avg_duration_per_task_ms": 105161.7, "avg_output_tokens_per_task": 15935.586666666666, "last_run_timestamp": "2026-10-24T11:51:53.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7488236470588235, "accuracy_std_dev": 0.024613068112981307, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.3698944605882353, "avg_duration_per_task_ms": 152128.24411764706, "avg_output_tokens_per_task": 11726.421176470589, "last_run_timestamp": "2026-10-31T12:39:28.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7485714285714286, "accuracy_std_dev": 0.023560522731632144, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.34892749714285715, "avg_duration_per_task_ms": 98456.37428571428, "avg_output_tokens_per_task": 7086.754285714285, "last_run_timestamp": "2026-10-31T11:27:20.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7485714285714286, "accuracy_std_dev": 0.04934029368103327, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11107009428571428, "avg_duration_per_task_ms": 240868.85714285713, "avg_output_tokens_per_task": 11857.671428571428, "last_run_timestamp": "2026-10-24T19:31:21.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.748484909090909, "accuracy_std_dev": 0.035629023673135175, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.19773472545454546, "avg_duration_per_task_ms": 96148.09090909091, "avg_output_tokens_per_task": 7605.130909090909, "last_run_timestamp": "2026-10-31T11:28:40.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.748333375, "accuracy_std_dev": 0.024664456546503833, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0092696725, "avg_duration_per_task_ms": 91481.9675, "avg_output_tokens_per_task": 13410.6275, "last_run_timestamp": "2026-10-31T13:03:50.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.748322067114094, "accuracy_std_dev": 0.02148614665313445, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.029814261744966444, "avg_duration_per_task_ms": 257681.3489932886, "avg_output_tokens_per_task": 10900.818791946309, "last_run_timestamp": "2026-10-31T12:26:05.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7481482222222222, "accuracy_std_dev": 0.020068464436510978, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19089935333333333, "avg_duration_per_task_ms": 286228.22222222225, "avg_output_tokens_per_task": 11572.273333333333, "last_run_timestamp": "2026-10-31T12:37:16.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.74791675, "accuracy_std_dev": 0.031222055284486006, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01487202, "avg_duration_per_task_ms": 51526.7675, "avg_output_tokens_per_task": 4120.2525, "last_run_timestamp": "2026-10-31T13:13:13.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7478510515759312, "accuracy_std_dev": 0.031000808296901307, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.018540833810888252, "avg_duration_per_task_ms": 126193.2005730659, "avg_output_tokens_per_task": 4035.12893982808, "last_run_timestamp": "2026-10-31T11:27:58.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.7477776666666667, "accuracy_std_dev": 0.028393017748891882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020487799999999997, "avg_duration_per_task_ms": 112394.16, "avg_output_tokens_per_task": 6879.653333333334, "last_run_timestamp": "2026-10-31T12:42:03.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7476435861182519, "accuracy_std_dev": 0.0440966339389981, "run_count": 8, "total_questions": 389, "avg_cost_per_task_usd": 0.10830866066838046, "avg_duration_per_task_ms": 421756.3444730077, "avg_output_tokens_per_task": 20888.023136246786, "last_run_timestamp": "2026-10-31T19:29:13.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7474576237288135, "accuracy_std_dev": 0.020378547413068143, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.010754061016949153, "avg_duration_per_task_ms": 127184.37966101695, "avg_output_tokens_per_task": 12239.813559322034, "last_run_timestamp": "2026-10-24T19:21:18.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7473684210526316, "accuracy_std_dev": 0.02465141176979054, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.02123823368421052, "avg_duration_per_task_ms": 106810.42210526316, "avg_output_tokens_per_task": 14307.62947368421, "last_run_timestamp": "2026-10-31T13:23:12.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7472727272727273, "accuracy_std_dev": 0.024197018843767105, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5306860090909091, "avg_duration_per_task_ms": 186964.4309090909, "avg_output_tokens_per_task": 10402.76909090909, "last_run_timestamp": "2026-10-31T11:53:29.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7472727272727273, "accuracy_std_dev": 0.02643158433008411, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.018947145454545452, "avg_duration_per_task_ms": 214213.9818181818, "avg_output_tokens_per_task": 15639.545454545454, "last_run_timestamp": "2026-10-31T12:32:12.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7472221666666666, "accuracy_std_dev": 0.011613686586906374, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17439429333333334, "avg_duration_per_task_ms": 161420.59666666668, "avg_output_tokens_per_task": 12499.9, "last_run_timestamp": "2026-10-10T11:19:48.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.746667, "accuracy_std_dev": 0.016329931618554536, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020258, "avg_duration_per_task_ms": 489464.5733333333, "avg_output_tokens_per_task": 14818, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00930136, "avg_duration_per_task_ms": 89811.58, "avg_output_tokens_per_task": 12020.36, "last_run_timestamp": "2026-10-24T15:18:33.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07114796, "avg_duration_per_task_ms": 129363.94, "avg_output_tokens_per_task": 8164.92, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Io Net", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.473986, "avg_duration_per_task_ms": 617970.5, "avg_output_tokens_per_task": 17752, "last_run_timestamp": "2026-11-04T17:25:46.000Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "DeepInfra", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.22191687999999998, "avg_duration_per_task_ms": 2285676.56, "avg_output_tokens_per_task": 10495.84, "last_run_timestamp": "2026-11-05T06:48:08.646Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "AkashML", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21601631999999998, "avg_duration_per_task_ms": 312892.28, "avg_output_tokens_per_task": 10774.7, "last_run_timestamp": "2026-11-05T06:21:39.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Alibaba", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.33231256000000003, "avg_duration_per_task_ms": 225567.04, "avg_output_tokens_per_task": 11123.54, "last_run_timestamp": "2026-11-04T21:00:38.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.070158785, "avg_duration_per_task_ms": 1726878.96, "avg_output_tokens_per_task": 17573.88, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.022607717464120523, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06205405, "avg_duration_per_task_ms": 201734.45, "avg_output_tokens_per_task": 11582.5475, "last_run_timestamp": "2026-10-31T19:58:43.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7466667272727272, "accuracy_std_dev": 0.01969470010430084, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3479023018181817, "avg_duration_per_task_ms": 215177.51454545456, "avg_output_tokens_per_task": 11904.012727272728, "last_run_timestamp": "2026-10-31T13:31:12.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7466665625000001, "accuracy_std_dev": 0.022360772946078904, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.013514558749999999, "avg_duration_per_task_ms": 139486.7775, "avg_output_tokens_per_task": 15486.175, "last_run_timestamp": "2026-10-31T13:03:01.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.7466665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00520861, "avg_duration_per_task_ms": 148091.91, "avg_output_tokens_per_task": 12193.49, "last_run_timestamp": "2026-10-31T11:47:33.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.7466663333333334, "accuracy_std_dev": 0.033993463423951875, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01425808, "avg_duration_per_task_ms": 139346.08, "avg_output_tokens_per_task": 12850.746666666666, "last_run_timestamp": "2026-10-31T12:18:46.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7460605454545455, "accuracy_std_dev": 0.03392861625494335, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03215642727272727, "avg_duration_per_task_ms": 233795.59636363637, "avg_output_tokens_per_task": 16719.21090909091, "last_run_timestamp": "2026-10-31T12:25:12.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.745833375, "accuracy_std_dev": 0.03307177485007986, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0937693475, "avg_duration_per_task_ms": 255498.785, "avg_output_tokens_per_task": 6773.36, "last_run_timestamp": "2026-10-31T19:33:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.74583325, "accuracy_std_dev": 0.03402396010442498, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02499106, "avg_duration_per_task_ms": 929627.78, "avg_output_tokens_per_task": 52531.49, "last_run_timestamp": "2026-10-24T12:31:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7457911447811448, "accuracy_std_dev": 0.035718498385523795, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.0683820404040404, "avg_duration_per_task_ms": 479113.58585858584, "avg_output_tokens_per_task": 10595.451178451178, "last_run_timestamp": "2026-10-24T19:42:30.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.029692331634924993, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5093009542857143, "avg_duration_per_task_ms": 175938.91142857142, "avg_output_tokens_per_task": 7450.982857142857, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7454546363636363, "accuracy_std_dev": 0.027239077558379347, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02595640363636364, "avg_duration_per_task_ms": 149834.41636363635, "avg_output_tokens_per_task": 9437.201818181818, "last_run_timestamp": "2026-10-31T19:14:12.566Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7454545454545455, "accuracy_std_dev": 0.02753421112779068, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5068107745454545, "avg_duration_per_task_ms": 192436.82727272727, "avg_output_tokens_per_task": 10076.376363636364, "last_run_timestamp": "2026-10-31T12:10:56.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7453394630872483, "accuracy_std_dev": 0.030816218886236597, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.06666300223713646, "avg_duration_per_task_ms": 1014852.2460850112, "avg_output_tokens_per_task": 75497.7091722595, "last_run_timestamp": "2026-10-24T19:38:23.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7453333, "accuracy_std_dev": 0.02454934079379731, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.17119798400000003, "avg_duration_per_task_ms": 320387.726, "avg_output_tokens_per_task": 11023.856, "last_run_timestamp": "2026-10-31T12:25:36.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7451853333333333, "accuracy_std_dev": 0.031234315132203188, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01557802666666667, "avg_duration_per_task_ms": 147555.71333333335, "avg_output_tokens_per_task": 15042.41111111111, "last_run_timestamp": "2026-10-31T12:25:02.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.017716056438394953, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.21677278888888887, "avg_duration_per_task_ms": 284791.18666666665, "avg_output_tokens_per_task": 11557.691111111111, "last_run_timestamp": "2026-10-31T11:21:59.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.745, "accuracy_std_dev": 0.024664481886915845, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0069834499999999996, "avg_duration_per_task_ms": 152934.4, "avg_output_tokens_per_task": 12714.7475, "last_run_timestamp": "2026-10-31T19:25:36.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.745, "accuracy_std_dev": 0.03840572873934308, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.014579895, "avg_duration_per_task_ms": 156494.315, "avg_output_tokens_per_task": 14989.845, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.745, "accuracy_std_dev": 0.008660254037844393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.033786329999999996, "avg_duration_per_task_ms": 75426.93, "avg_output_tokens_per_task": 13154.035, "last_run_timestamp": "2026-11-02T21:36:49.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.745, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15284815, "avg_duration_per_task_ms": 494527.49, "avg_output_tokens_per_task": 28246.935, "last_run_timestamp": "2026-10-17T19:47:21.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.744999875, "accuracy_std_dev": 0.030867001888900313, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0114373625, "avg_duration_per_task_ms": 404697.66, "avg_output_tokens_per_task": 5754.185, "last_run_timestamp": "2026-10-31T12:24:48.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7448979591836735, "accuracy_std_dev": 0.02585427538338681, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.01656998775510204, "avg_duration_per_task_ms": 900521.2571428572, "avg_output_tokens_per_task": 22085.71020408163, "last_run_timestamp": "2026-11-05T09:28:04.757Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "PrimaLabs", "accuracy": 0.7447914583333333, "accuracy_std_dev": 0.005435000000000023, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.13105378125, "avg_duration_per_task_ms": 273825.1666666667, "avg_output_tokens_per_task": 8026.104166666667, "last_run_timestamp": "2026-11-06T15:59:08.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7446667, "accuracy_std_dev": 0.01956180291818729, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.055664519999999995, "avg_duration_per_task_ms": 332682.316, "avg_output_tokens_per_task": 14458.648, "last_run_timestamp": "2026-10-31T11:42:33.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02140772, "avg_duration_per_task_ms": 53184.22, "avg_output_tokens_per_task": 14364.18, "last_run_timestamp": "2026-10-31T11:41:11.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7442423636363636, "accuracy_std_dev": 0.025784247154460715, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5240637272727273, "avg_duration_per_task_ms": 222945.58545454545, "avg_output_tokens_per_task": 9999.456363636364, "last_run_timestamp": "2026-10-31T13:21:45.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.74416675, "accuracy_std_dev": 0.04241809591657196, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11267837500000001, "avg_duration_per_task_ms": 196024.8775, "avg_output_tokens_per_task": 9614.775, "last_run_timestamp": "2026-10-31T20:27:27.677Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.744166625, "accuracy_std_dev": 0.03381937006708397, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0447137225, "avg_duration_per_task_ms": 180624.91, "avg_output_tokens_per_task": 9433.0525, "last_run_timestamp": "2026-10-31T20:15:42.614Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.744012664310954, "accuracy_std_dev": 0.013345937402400357, "run_count": 17, "total_questions": 849, "avg_cost_per_task_usd": 0.0449638492343934, "avg_duration_per_task_ms": 327500.66313309775, "avg_output_tokens_per_task": 26020.465253239105, "last_run_timestamp": "2026-10-31T19:20:57.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.74400005, "accuracy_std_dev": 0.022050473415496093, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.43035259, "avg_duration_per_task_ms": 608815.123, "avg_output_tokens_per_task": 10549.796, "last_run_timestamp": "2026-10-31T11:24:57.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.744, "accuracy_std_dev": 0.04013309523821955, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.12688716400000002, "avg_duration_per_task_ms": 700323.218, "avg_output_tokens_per_task": 36239.088, "last_run_timestamp": "2026-10-31T12:47:12.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7438595789473685, "accuracy_std_dev": 0.004074000000000022, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.022313326315789476, "avg_duration_per_task_ms": 210423.8947368421, "avg_output_tokens_per_task": 9630.8, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7436363636363637, "accuracy_std_dev": 0.025324294459427084, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.14332595090909092, "avg_duration_per_task_ms": 579280.3727272727, "avg_output_tokens_per_task": 26391.907272727272, "last_run_timestamp": "2026-10-31T19:26:38.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "PrimeIntellect", "accuracy": 0.743502793220339, "accuracy_std_dev": 0.018213736559866647, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.010024135593220337, "avg_duration_per_task_ms": 123315.84745762713, "avg_output_tokens_per_task": 13505.108474576271, "last_run_timestamp": "2026-11-02T17:51:25.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7431746190476191, "accuracy_std_dev": 0.018008818983714862, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.36043625523809525, "avg_duration_per_task_ms": 86437.01714285715, "avg_output_tokens_per_task": 7106.278095238095, "last_run_timestamp": "2026-10-31T12:25:51.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7430302727272727, "accuracy_std_dev": 0.027760033635986, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5088923236363636, "avg_duration_per_task_ms": 192892.78363636363, "avg_output_tokens_per_task": 9984.365454545454, "last_run_timestamp": "2026-10-31T12:42:53.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7430302272727273, "accuracy_std_dev": 0.0266459682076121, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5452918245454546, "avg_duration_per_task_ms": 200617.91363636364, "avg_output_tokens_per_task": 9759.658181818182, "last_run_timestamp": "2026-10-31T13:01:06.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7429168125, "accuracy_std_dev": 0.02570869361037903, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.05779907625, "avg_duration_per_task_ms": 136331.88625, "avg_output_tokens_per_task": 9998.7625, "last_run_timestamp": "2026-10-31T19:13:33.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7426667, "accuracy_std_dev": 0.028472143112347543, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.020416034, "avg_duration_per_task_ms": 153083.812, "avg_output_tokens_per_task": 16185.95, "last_run_timestamp": "2026-10-31T12:48:17.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.04434197309367279, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.16675142799999998, "avg_duration_per_task_ms": 302047.356, "avg_output_tokens_per_task": 9682.606, "last_run_timestamp": "2026-10-31T11:21:43.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7425926666666667, "accuracy_std_dev": 0.03165311687654157, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013586744444444443, "avg_duration_per_task_ms": 236886.98, "avg_output_tokens_per_task": 22693.746666666666, "last_run_timestamp": "2026-10-24T12:29:34.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7425405959079282, "accuracy_std_dev": 0.026283120985422872, "run_count": 8, "total_questions": 391, "avg_cost_per_task_usd": 0.12785004603580563, "avg_duration_per_task_ms": 203562.63938618926, "avg_output_tokens_per_task": 10880.268542199488, "last_run_timestamp": "2026-10-31T19:27:07.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.742500125, "accuracy_std_dev": 0.027271738271686614, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.041572885, "avg_duration_per_task_ms": 596372.7425, "avg_output_tokens_per_task": 15967.995, "last_run_timestamp": "2026-10-31T19:47:52.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7424243636363637, "accuracy_std_dev": 0.03232057912192878, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5451988545454546, "avg_duration_per_task_ms": 180223.92, "avg_output_tokens_per_task": 10245.276363636363, "last_run_timestamp": "2026-10-31T12:28:38.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7424241818181818, "accuracy_std_dev": 0.03530779679235989, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.36784247818181814, "avg_duration_per_task_ms": 381853.54363636364, "avg_output_tokens_per_task": 9773.532727272728, "last_run_timestamp": "2026-10-31T12:43:33.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.27693500666666665, "avg_duration_per_task_ms": 100083.4, "avg_output_tokens_per_task": 5197.013333333333, "last_run_timestamp": "2026-10-30T00:15:31.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.011330936570096763, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01793052666666667, "avg_duration_per_task_ms": 86484.18, "avg_output_tokens_per_task": 20778.286666666667, "last_run_timestamp": "2026-10-31T11:37:00.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.03456997912768952, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06651346666666666, "avg_duration_per_task_ms": 1404270.7533333334, "avg_output_tokens_per_task": 9023.02, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7420636446886446, "accuracy_std_dev": 0.016469967104193704, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.01720449084249084, "avg_duration_per_task_ms": 82040.02197802198, "avg_output_tokens_per_task": 4114.483516483517, "last_run_timestamp": "2026-10-31T12:00:42.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7420434974874371, "accuracy_std_dev": 0.018804943831343932, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.046914160804020104, "avg_duration_per_task_ms": 685430.3140703518, "avg_output_tokens_per_task": 14198.173366834171, "last_run_timestamp": "2026-10-31T19:48:13.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7418518888888889, "accuracy_std_dev": 0.020793479247347656, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3703597533333333, "avg_duration_per_task_ms": 185758.07111111112, "avg_output_tokens_per_task": 10963.044444444444, "last_run_timestamp": "2026-10-31T11:51:20.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.741666625, "accuracy_std_dev": 0.025331164930464133, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021842495, "avg_duration_per_task_ms": 141327.99, "avg_output_tokens_per_task": 16250.775, "last_run_timestamp": "2026-10-31T12:24:54.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.741666625, "accuracy_std_dev": 0.030866893899198455, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021115824999999998, "avg_duration_per_task_ms": 150544.0025, "avg_output_tokens_per_task": 8219.325, "last_run_timestamp": "2026-10-31T18:57:59.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7416481202672606, "accuracy_std_dev": 0.022563192218990436, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.21138012694877506, "avg_duration_per_task_ms": 402109.59020044544, "avg_output_tokens_per_task": 9801.538975501113, "last_run_timestamp": "2026-10-31T11:26:03.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7416426103151863, "accuracy_std_dev": 0.022067987204666104, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.051895842406876794, "avg_duration_per_task_ms": 765180.4412607449, "avg_output_tokens_per_task": 14064.899713467048, "last_run_timestamp": "2026-10-24T19:40:41.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.7414965918367348, "accuracy_std_dev": 0.01208300000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.014484438775510205, "avg_duration_per_task_ms": 90891.24489795919, "avg_output_tokens_per_task": 9684.755102040815, "last_run_timestamp": "2026-10-31T13:18:30.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7414814444444444, "accuracy_std_dev": 0.026810305535625297, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02054646222222222, "avg_duration_per_task_ms": 418578.0088888889, "avg_output_tokens_per_task": 11914.037777777778, "last_run_timestamp": "2026-10-17T11:29:07.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.74136442746114, "accuracy_std_dev": 0.023598813561278888, "run_count": 8, "total_questions": 386, "avg_cost_per_task_usd": 0.1499042176165803, "avg_duration_per_task_ms": 675715.4041450777, "avg_output_tokens_per_task": 29694.012953367877, "last_run_timestamp": "2026-10-31T19:55:58.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.7413334, "accuracy_std_dev": 0.02436751571744648, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.097602524, "avg_duration_per_task_ms": 217804.16, "avg_output_tokens_per_task": 9920.02, "last_run_timestamp": "2026-10-31T14:40:25.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7413333, "accuracy_std_dev": 0.03095523075685272, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.40647999399999996, "avg_duration_per_task_ms": 588538.78, "avg_output_tokens_per_task": 9896.416, "last_run_timestamp": "2026-10-31T12:27:47.241Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7412121818181818, "accuracy_std_dev": 0.0275340250329006, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3449351727272727, "avg_duration_per_task_ms": 200135.41454545455, "avg_output_tokens_per_task": 11800.205454545454, "last_run_timestamp": "2026-10-31T13:12:27.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.7410254102564102, "accuracy_std_dev": 0.015687179899124656, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.010222671794871796, "avg_duration_per_task_ms": 114819.93846153846, "avg_output_tokens_per_task": 3734.005128205128, "last_run_timestamp": "2026-10-31T12:06:01.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7409522857142857, "accuracy_std_dev": 0.024018230038953373, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05641216857142857, "avg_duration_per_task_ms": 852526.1028571428, "avg_output_tokens_per_task": 15826, "last_run_timestamp": "2026-10-31T20:02:53.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7408025986622073, "accuracy_std_dev": 0.026595914006744867, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.04194225585284281, "avg_duration_per_task_ms": 163070.16889632106, "avg_output_tokens_per_task": 8777.19397993311, "last_run_timestamp": "2026-10-31T19:46:31.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7406666000000001, "accuracy_std_dev": 0.018961987708043703, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014243233999999999, "avg_duration_per_task_ms": 72756.944, "avg_output_tokens_per_task": 4429.298, "last_run_timestamp": "2026-10-31T12:42:18.186Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7406060909090909, "accuracy_std_dev": 0.026123868248489987, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3297550236363636, "avg_duration_per_task_ms": 195835.06181818183, "avg_output_tokens_per_task": 11555.365454545454, "last_run_timestamp": "2026-10-31T12:07:21.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7403698151001541, "accuracy_std_dev": 0.028547774310492945, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.012115494607087829, "avg_duration_per_task_ms": 425113.8289676425, "avg_output_tokens_per_task": 14181.97688751926, "last_run_timestamp": "2026-10-31T11:58:45.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.74014702004008, "accuracy_std_dev": 0.022155950173486136, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.06005350100200401, "avg_duration_per_task_ms": 140448.37274549098, "avg_output_tokens_per_task": 4119.851703406814, "last_run_timestamp": "2026-10-31T12:27:45.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7400002499999999, "accuracy_std_dev": 0.029999944444740217, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08284509749999999, "avg_duration_per_task_ms": 74658.595, "avg_output_tokens_per_task": 11791.0375, "last_run_timestamp": "2026-10-31T19:56:16.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400001666666666, "accuracy_std_dev": 0.01154690915954955, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.034900296666666664, "avg_duration_per_task_ms": 129527.63333333333, "avg_output_tokens_per_task": 16132.923333333334, "last_run_timestamp": "2026-10-27T00:42:49.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7400001111111111, "accuracy_std_dev": 0.024545206467724207, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07987433111111111, "avg_duration_per_task_ms": 314745.96222222224, "avg_output_tokens_per_task": 11321.113333333333, "last_run_timestamp": "2026-10-31T21:26:40.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.23335095, "avg_duration_per_task_ms": 96091.26, "avg_output_tokens_per_task": 5555.9, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.74, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14729807, "avg_duration_per_task_ms": 386775.27, "avg_output_tokens_per_task": 24280.02, "last_run_timestamp": "2026-10-31T19:58:44.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.74, "accuracy_std_dev": 0.03511903567511501, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.023083885, "avg_duration_per_task_ms": 150345.7, "avg_output_tokens_per_task": 16221.6925, "last_run_timestamp": "2026-10-31T11:30:43.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Near AI", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01862194, "avg_duration_per_task_ms": 339225.77, "avg_output_tokens_per_task": 12354.38, "last_run_timestamp": "2026-10-31T13:22:50.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.37915447, "avg_duration_per_task_ms": 148084.91, "avg_output_tokens_per_task": 12525.4, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00321776, "avg_duration_per_task_ms": 67631.78, "avg_output_tokens_per_task": 11380.95, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.74, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06914866, "avg_duration_per_task_ms": 152605.49, "avg_output_tokens_per_task": 14774.68, "last_run_timestamp": "2026-10-31T12:25:52.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1350024, "avg_duration_per_task_ms": 461018.86, "avg_output_tokens_per_task": 26035.68, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.74, "accuracy_std_dev": 0.03471220655619578, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02934289777777778, "avg_duration_per_task_ms": 461721.2688888889, "avg_output_tokens_per_task": 15007.28, "last_run_timestamp": "2026-10-17T13:55:17.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.74, "accuracy_std_dev": 0.019860653454128848, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.01624762875, "avg_duration_per_task_ms": 161764.35125, "avg_output_tokens_per_task": 4237.58, "last_run_timestamp": "2026-10-31T11:33:42.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7399999090909091, "accuracy_std_dev": 0.02785246847377528, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3035846727272727, "avg_duration_per_task_ms": 295316.7327272727, "avg_output_tokens_per_task": 9413.458181818181, "last_run_timestamp": "2026-10-31T13:04:39.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7399329261744967, "accuracy_std_dev": 0.017150616176167584, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.24555212751677855, "avg_duration_per_task_ms": 357614.23825503356, "avg_output_tokens_per_task": 12408.355704697986, "last_run_timestamp": "2026-10-31T12:24:39.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7395555333333333, "accuracy_std_dev": 0.016947170784004716, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013297554666666666, "avg_duration_per_task_ms": 100947.12933333333, "avg_output_tokens_per_task": 10746.964, "last_run_timestamp": "2026-10-31T12:45:41.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7394444166666667, "accuracy_std_dev": 0.02168094045645598, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.3449865733333333, "avg_duration_per_task_ms": 207685.65666666668, "avg_output_tokens_per_task": 11866.296666666667, "last_run_timestamp": "2026-10-31T13:36:32.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7393938181818183, "accuracy_std_dev": 0.027627475312607935, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.34960052363636374, "avg_duration_per_task_ms": 219975.07454545455, "avg_output_tokens_per_task": 10861.59090909091, "last_run_timestamp": "2026-10-31T12:46:18.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7392308461538462, "accuracy_std_dev": 0.0224239080065011, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.017090104615384617, "avg_duration_per_task_ms": 299674.4630769231, "avg_output_tokens_per_task": 22554.26307692308, "last_run_timestamp": "2026-11-04T21:17:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.73916675, "accuracy_std_dev": 0.032220958069826236, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1383197475, "avg_duration_per_task_ms": 366177.2025, "avg_output_tokens_per_task": 11823.9525, "last_run_timestamp": "2026-10-31T21:07:15.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.738889, "accuracy_std_dev": 0.024695368688615833, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01095846, "avg_duration_per_task_ms": 321436.35333333333, "avg_output_tokens_per_task": 22570.04, "last_run_timestamp": "2026-10-31T12:52:06.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7386668, "accuracy_std_dev": 0.03166503013357166, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009092168000000001, "avg_duration_per_task_ms": 101935.708, "avg_output_tokens_per_task": 5027.116, "last_run_timestamp": "2026-10-17T11:17:02.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7386667, "accuracy_std_dev": 0.04987086078673599, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009059300000000001, "avg_duration_per_task_ms": 243713.362, "avg_output_tokens_per_task": 15953.852, "last_run_timestamp": "2026-10-31T11:40:30.784Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7385858181818183, "accuracy_std_dev": 0.02930192686750755, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.3660526951515151, "avg_duration_per_task_ms": 198575.79333333333, "avg_output_tokens_per_task": 11916.818181818182, "last_run_timestamp": "2026-10-31T13:17:20.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.025293917789377812, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.008203653333333333, "avg_duration_per_task_ms": 425108.4222222222, "avg_output_tokens_per_task": 14620.724444444444, "last_run_timestamp": "2026-10-17T12:09:02.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7383335, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00751252, "avg_duration_per_task_ms": 320365.75, "avg_output_tokens_per_task": 15536.72, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7383335, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08170183999999998, "avg_duration_per_task_ms": 327224.56, "avg_output_tokens_per_task": 8775.96, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.738333375, "accuracy_std_dev": 0.03244649655948039, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.049700385, "avg_duration_per_task_ms": 377861.96, "avg_output_tokens_per_task": 11236.51, "last_run_timestamp": "2026-10-31T21:59:10.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.73833325, "accuracy_std_dev": 0.03460099147694328, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0732567425, "avg_duration_per_task_ms": 220285.0375, "avg_output_tokens_per_task": 9565.1675, "last_run_timestamp": "2026-10-31T19:53:34.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7382021797752809, "accuracy_std_dev": 0.021880590311850887, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.020498339325842697, "avg_duration_per_task_ms": 307841.11235955055, "avg_output_tokens_per_task": 10994.101123595505, "last_run_timestamp": "2026-10-17T11:09:59.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7381817272727274, "accuracy_std_dev": 0.026375810495400075, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0789221490909091, "avg_duration_per_task_ms": 120883.72, "avg_output_tokens_per_task": 8767.690909090908, "last_run_timestamp": "2026-10-31T20:18:12.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7380954285714285, "accuracy_std_dev": 0.030801812534131074, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0719204, "avg_duration_per_task_ms": 327417.77714285714, "avg_output_tokens_per_task": 11436.125714285714, "last_run_timestamp": "2026-10-24T22:27:21.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7380953482142857, "accuracy_std_dev": 0.031652037557723764, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.004402678571428571, "avg_duration_per_task_ms": 256685.421875, "avg_output_tokens_per_task": 12156.511160714286, "last_run_timestamp": "2026-10-31T19:49:31.140Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7380952719449226, "accuracy_std_dev": 0.03197285191855047, "run_count": 12, "total_questions": 581, "avg_cost_per_task_usd": 0.024856838209982787, "avg_duration_per_task_ms": 93869.10499139415, "avg_output_tokens_per_task": 8326.619621342514, "last_run_timestamp": "2026-11-06T06:42:03.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7380000999999999, "accuracy_std_dev": 0.02844855708274146, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.016056571999999998, "avg_duration_per_task_ms": 534586.188, "avg_output_tokens_per_task": 17634.696, "last_run_timestamp": "2026-10-31T12:03:50.388Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.737882, "accuracy_std_dev": 0.023142126657293677, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.3130514742729307, "avg_duration_per_task_ms": 411436.096196868, "avg_output_tokens_per_task": 9782.814317673377, "last_run_timestamp": "2026-10-24T12:29:09.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.027933050471917086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02205162, "avg_duration_per_task_ms": 230910.26, "avg_output_tokens_per_task": 11297.92, "last_run_timestamp": "2026-10-31T12:27:40.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009694766666666667, "avg_duration_per_task_ms": 179217.14666666667, "avg_output_tokens_per_task": 12873.993333333334, "last_run_timestamp": "2026-10-31T11:06:39.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.044333155109917456, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.010008253333333335, "avg_duration_per_task_ms": 438344.2577777778, "avg_output_tokens_per_task": 14883.606666666667, "last_run_timestamp": "2026-10-17T11:26:35.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7377777777777778, "accuracy_std_dev": 0.03189484265094273, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06719376444444446, "avg_duration_per_task_ms": 260527.6288888889, "avg_output_tokens_per_task": 9620.40888888889, "last_run_timestamp": "2026-10-31T20:31:43.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7377776666666667, "accuracy_std_dev": 0.020366893981273733, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3511924911111111, "avg_duration_per_task_ms": 195777.59333333332, "avg_output_tokens_per_task": 9122.282222222222, "last_run_timestamp": "2026-10-31T11:28:52.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7376126486486487, "accuracy_std_dev": 0.027927098584230564, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.011551577702702704, "avg_duration_per_task_ms": 61368.75337837838, "avg_output_tokens_per_task": 4262.564189189189, "last_run_timestamp": "2026-10-31T12:25:51.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.737499875, "accuracy_std_dev": 0.022592425906913488, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01732587, "avg_duration_per_task_ms": 193821.065, "avg_output_tokens_per_task": 4493.1475, "last_run_timestamp": "2026-10-31T12:42:00.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7373334, "accuracy_std_dev": 0.020482933819157834, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039714615999999994, "avg_duration_per_task_ms": 174773.632, "avg_output_tokens_per_task": 27822.08, "last_run_timestamp": "2026-10-21T12:36:51.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7371427142857143, "accuracy_std_dev": 0.04561487206325298, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.006840197142857143, "avg_duration_per_task_ms": 433214.3028571429, "avg_output_tokens_per_task": 14593.82, "last_run_timestamp": "2026-10-10T15:31:23.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7370371111111111, "accuracy_std_dev": 0.03175710369680895, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024656559999999994, "avg_duration_per_task_ms": 423744.3888888889, "avg_output_tokens_per_task": 15866.275555555556, "last_run_timestamp": "2026-10-17T11:15:55.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7370184522613066, "accuracy_std_dev": 0.037544250024970534, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.07583668341708542, "avg_duration_per_task_ms": 861965.7211055276, "avg_output_tokens_per_task": 14681.494974874371, "last_run_timestamp": "2026-10-24T20:35:30.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.47698523000000004, "avg_duration_per_task_ms": 128774.34, "avg_output_tokens_per_task": 7206.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7366665, "accuracy_std_dev": 0.02285212122539177, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0637652175, "avg_duration_per_task_ms": 239799.505, "avg_output_tokens_per_task": 12644.0625, "last_run_timestamp": "2026-10-31T20:18:31.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02539905, "avg_duration_per_task_ms": 216631.23, "avg_output_tokens_per_task": 14336.27, "last_run_timestamp": "2026-10-24T12:02:36.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7363637272727273, "accuracy_std_dev": 0.029727967744168908, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.029319285454545447, "avg_duration_per_task_ms": 101893.45090909091, "avg_output_tokens_per_task": 10418.469090909091, "last_run_timestamp": "2026-10-31T19:09:49.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7362762003395587, "accuracy_std_dev": 0.03269045093786308, "run_count": 12, "total_questions": 589, "avg_cost_per_task_usd": 0.06555029541595926, "avg_duration_per_task_ms": 138699.1324278438, "avg_output_tokens_per_task": 8434.507640067912, "last_run_timestamp": "2026-10-31T19:10:53.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.736, "accuracy_std_dev": 0.011623577573191504, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.02217724, "avg_duration_per_task_ms": 74896.184, "avg_output_tokens_per_task": 4355.024, "last_run_timestamp": "2026-10-31T12:05:37.260Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.7359999333333334, "accuracy_std_dev": 0.0231964569463145, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013030309333333335, "avg_duration_per_task_ms": 73246.75466666666, "avg_output_tokens_per_task": 10779.536, "last_run_timestamp": "2026-10-31T11:42:11.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7359999, "accuracy_std_dev": 0.02831560386588993, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.260319256, "avg_duration_per_task_ms": 297852.642, "avg_output_tokens_per_task": 10753.23, "last_run_timestamp": "2026-10-31T11:38:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7359261111111111, "accuracy_std_dev": 0.02318882707227026, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04132428666666666, "avg_duration_per_task_ms": 135748.28, "avg_output_tokens_per_task": 25779.64, "last_run_timestamp": "2026-10-31T12:16:10.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.735833375, "accuracy_std_dev": 0.017618592217722016, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11823662750000001, "avg_duration_per_task_ms": 341338.0975, "avg_output_tokens_per_task": 9727.67, "last_run_timestamp": "2026-10-31T20:34:53.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "NextBit", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.01369867976201439, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01667712, "avg_duration_per_task_ms": 656503.2166666667, "avg_output_tokens_per_task": 11802.333333333334, "last_run_timestamp": "2026-11-03T00:15:40.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.04157400066013479, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48569376, "avg_duration_per_task_ms": 125885.67333333334, "avg_output_tokens_per_task": 7296.606666666667, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7355555555555555, "accuracy_std_dev": 0.03220320511692074, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04873943777777778, "avg_duration_per_task_ms": 232537.14444444445, "avg_output_tokens_per_task": 10149.82, "last_run_timestamp": "2026-10-31T19:47:34.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7355555, "accuracy_std_dev": 0.037646835160253966, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.009423405555555554, "avg_duration_per_task_ms": 235235.8688888889, "avg_output_tokens_per_task": 15545.602222222222, "last_run_timestamp": "2026-11-02T18:06:38.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7355553333333333, "accuracy_std_dev": 0.02997928502964824, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008365386666666667, "avg_duration_per_task_ms": 195637.39333333334, "avg_output_tokens_per_task": 12883.673333333334, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7355553333333333, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05735756666666667, "avg_duration_per_task_ms": 219541.36, "avg_output_tokens_per_task": 9149.013333333334, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7352380000000001, "accuracy_std_dev": 0.02932336720092016, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.011415382857142857, "avg_duration_per_task_ms": 224754.26285714286, "avg_output_tokens_per_task": 14326.351428571428, "last_run_timestamp": "2026-10-31T19:46:05.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7352380000000001, "accuracy_std_dev": 0.01698343330593503, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3052330542857143, "avg_duration_per_task_ms": 191492.84285714285, "avg_output_tokens_per_task": 10672.025714285714, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.735000125, "accuracy_std_dev": 0.04186617734650937, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0273002175, "avg_duration_per_task_ms": 144987.4475, "avg_output_tokens_per_task": 12792.885, "last_run_timestamp": "2026-10-31T18:53:34.804Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7349398099062917, "accuracy_std_dev": 0.02455646627618342, "run_count": 15, "total_questions": 747, "avg_cost_per_task_usd": 0.05549164792503348, "avg_duration_per_task_ms": 157974.1485943775, "avg_output_tokens_per_task": 30263.555555555555, "last_run_timestamp": "2026-10-31T12:41:59.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7349397911646587, "accuracy_std_dev": 0.047428285455095254, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.2969604819277108, "avg_duration_per_task_ms": 386565.9397590361, "avg_output_tokens_per_task": 11351.530120481928, "last_run_timestamp": "2026-10-31T12:42:56.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7344445, "accuracy_std_dev": 0.0325272326558839, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07984409666666667, "avg_duration_per_task_ms": 507527.12666666665, "avg_output_tokens_per_task": 12020.823333333334, "last_run_timestamp": "2026-10-24T23:08:37.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.734166625, "accuracy_std_dev": 0.031169207785639575, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0986136275, "avg_duration_per_task_ms": 266155.0625, "avg_output_tokens_per_task": 12567.385, "last_run_timestamp": "2026-10-31T20:07:04.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7341136153846154, "accuracy_std_dev": 0.026553848166927203, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.12951078260869564, "avg_duration_per_task_ms": 62609.57525083612, "avg_output_tokens_per_task": 9507.267558528429, "last_run_timestamp": "2026-10-31T19:12:18.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.733796, "accuracy_std_dev": 0.0059017228557995535, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.20467558333333333, "avg_duration_per_task_ms": 370954.7361111111, "avg_output_tokens_per_task": 23212.9375, "last_run_timestamp": "2026-10-24T19:18:46.938Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7337547565858799, "accuracy_std_dev": 0.027618715667845153, "run_count": 19, "total_questions": 949, "avg_cost_per_task_usd": 0.12936480400421493, "avg_duration_per_task_ms": 156235.47629083245, "avg_output_tokens_per_task": 9321.471022128557, "last_run_timestamp": "2026-10-24T19:48:18.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7335003934837092, "accuracy_std_dev": 0.03251829565197682, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.06496012280701755, "avg_duration_per_task_ms": 149415.91228070174, "avg_output_tokens_per_task": 11151.812030075187, "last_run_timestamp": "2026-10-31T19:24:16.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12301775, "avg_duration_per_task_ms": 155307.55, "avg_output_tokens_per_task": 18812.75, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.7333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08901925000000001, "avg_duration_per_task_ms": 93612.32, "avg_output_tokens_per_task": 8634.62, "last_run_timestamp": "2026-10-31T11:29:29.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08394800999999999, "avg_duration_per_task_ms": 134148.76, "avg_output_tokens_per_task": 8723.94, "last_run_timestamp": "2026-10-17T19:55:39.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7333335, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12687428, "avg_duration_per_task_ms": 110480.49, "avg_output_tokens_per_task": 5152.54, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.7333334, "accuracy_std_dev": 0.03451234883690184, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.30572972400000004, "avg_duration_per_task_ms": 282521.796, "avg_output_tokens_per_task": 10813.856, "last_run_timestamp": "2026-10-31T12:51:11.835Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.733333375, "accuracy_std_dev": 0.027688655921412557, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.051658964999999994, "avg_duration_per_task_ms": 354319.435, "avg_output_tokens_per_task": 10072.31, "last_run_timestamp": "2026-10-31T21:01:26.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06451945333333334, "avg_duration_per_task_ms": 876085.0533333333, "avg_output_tokens_per_task": 15144.64, "last_run_timestamp": "2026-10-31T12:31:12.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.03569390445004426, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021479246666666667, "avg_duration_per_task_ms": 120781.28, "avg_output_tokens_per_task": 11581.666666666666, "last_run_timestamp": "2026-10-31T11:22:01.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05760637824832321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04898614, "avg_duration_per_task_ms": 106829.63333333333, "avg_output_tokens_per_task": 9897.326666666666, "last_run_timestamp": "2026-10-10T11:51:48.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09260214, "avg_duration_per_task_ms": 326558.9733333333, "avg_output_tokens_per_task": 4729.153333333334, "last_run_timestamp": "2026-10-31T12:28:59.426Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024898453333333334, "avg_duration_per_task_ms": 37126.9, "avg_output_tokens_per_task": 4153.473333333333, "last_run_timestamp": "2026-10-31T12:07:50.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.73333325, "accuracy_std_dev": 0.0368176738970226, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013768695000000003, "avg_duration_per_task_ms": 81021.62, "avg_output_tokens_per_task": 13622.935, "last_run_timestamp": "2026-11-01T12:08:57.072Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.7333332, "accuracy_std_dev": 0.032659904062320846, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012294704000000002, "avg_duration_per_task_ms": 85675.876, "avg_output_tokens_per_task": 10698.828, "last_run_timestamp": "2026-10-31T11:35:21.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7333332, "accuracy_std_dev": 0.03211094655970144, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.18171748399999998, "avg_duration_per_task_ms": 476804.58, "avg_output_tokens_per_task": 9681.516, "last_run_timestamp": "2026-10-31T22:21:14.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DekaLLM", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01115704, "avg_duration_per_task_ms": 102700.12, "avg_output_tokens_per_task": 12077.2, "last_run_timestamp": "2026-11-04T09:13:50.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12365041999999998, "avg_duration_per_task_ms": 156560.04, "avg_output_tokens_per_task": 19704.13, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10928178, "avg_duration_per_task_ms": 75479.44, "avg_output_tokens_per_task": 18824.05, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Parasail", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2484202, "avg_duration_per_task_ms": 207733.62, "avg_output_tokens_per_task": 10608.24, "last_run_timestamp": "2026-11-04T21:15:40.182Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Together", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2865894, "avg_duration_per_task_ms": 249378.06, "avg_output_tokens_per_task": 16191.68, "last_run_timestamp": "2026-11-04T18:44:37.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.732916625, "accuracy_std_dev": 0.018888392049996622, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0482453475, "avg_duration_per_task_ms": 162563.185, "avg_output_tokens_per_task": 29218.5525, "last_run_timestamp": "2026-10-31T12:25:59.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.7327778333333334, "accuracy_std_dev": 0.03057555204416685, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01934296, "avg_duration_per_task_ms": 97612.35666666667, "avg_output_tokens_per_task": 5244.636666666666, "last_run_timestamp": "2026-11-02T21:39:06.869Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7326191428571428, "accuracy_std_dev": 0.015745039894052536, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.2114459285714286, "avg_duration_per_task_ms": 238357.1, "avg_output_tokens_per_task": 9801.7, "last_run_timestamp": "2026-11-06T22:35:15.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7325, "accuracy_std_dev": 0.04596937990662913, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012367795, "avg_duration_per_task_ms": 953069.875, "avg_output_tokens_per_task": 22737, "last_run_timestamp": "2026-10-17T10:32:07.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7324560809716599, "accuracy_std_dev": 0.04384289047314286, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.02773825708502024, "avg_duration_per_task_ms": 111349.57489878543, "avg_output_tokens_per_task": 10355.305668016194, "last_run_timestamp": "2026-10-31T19:48:53.852Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.732, "accuracy_std_dev": 0.02124973202654565, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036015575999999994, "avg_duration_per_task_ms": 148623.836, "avg_output_tokens_per_task": 32577.608, "last_run_timestamp": "2026-10-31T12:40:24.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.7316665, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01274185, "avg_duration_per_task_ms": 154997.2, "avg_output_tokens_per_task": 13696.96, "last_run_timestamp": "2026-10-31T19:54:14.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.7313332, "accuracy_std_dev": 0.020231941744676914, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.012560144, "avg_duration_per_task_ms": 74716.332, "avg_output_tokens_per_task": 10525.228, "last_run_timestamp": "2026-10-31T11:19:23.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7311827782258065, "accuracy_std_dev": 0.07837008853695393, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.009883709677419354, "avg_duration_per_task_ms": 170348.1431451613, "avg_output_tokens_per_task": 4275.5161290322585, "last_run_timestamp": "2026-10-31T12:31:51.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7311637573964497, "accuracy_std_dev": 0.02813377440238986, "run_count": 17, "total_questions": 845, "avg_cost_per_task_usd": 0.024811487573964494, "avg_duration_per_task_ms": 209572.125443787, "avg_output_tokens_per_task": 21468.289940828403, "last_run_timestamp": "2026-10-31T11:27:16.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.7311111666666666, "accuracy_std_dev": 0.03022542058387644, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05052855666666667, "avg_duration_per_task_ms": 436577.48333333334, "avg_output_tokens_per_task": 9693.223333333333, "last_run_timestamp": "2026-10-24T20:04:11.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06301382666666666, "avg_duration_per_task_ms": 511947.82666666666, "avg_output_tokens_per_task": 14587.22, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028709900000000003, "avg_duration_per_task_ms": 638312.6933333334, "avg_output_tokens_per_task": 61738.18, "last_run_timestamp": "2026-10-31T12:28:49.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12928292, "avg_duration_per_task_ms": 109953.54, "avg_output_tokens_per_task": 5131.753333333333, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.730909090909091, "accuracy_std_dev": 0.04442884014497901, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02853827818181818, "avg_duration_per_task_ms": 115537.2709090909, "avg_output_tokens_per_task": 9990.294545454546, "last_run_timestamp": "2026-10-31T19:22:00.212Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.73083325, "accuracy_std_dev": 0.029988453797878633, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.097229255, "avg_duration_per_task_ms": 318293.035, "avg_output_tokens_per_task": 11028.215, "last_run_timestamp": "2026-10-31T20:47:51.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7306667, "accuracy_std_dev": 0.06533322244930216, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.15851144199999997, "avg_duration_per_task_ms": 1396906.662, "avg_output_tokens_per_task": 28393.436, "last_run_timestamp": "2026-10-24T20:45:53.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.730303, "accuracy_std_dev": 0.02426511211221128, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.364338510909091, "avg_duration_per_task_ms": 226168.5118181818, "avg_output_tokens_per_task": 10866.631818181819, "last_run_timestamp": "2026-10-31T12:43:31.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7300002499999999, "accuracy_std_dev": 0.028867484592314224, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03407485, "avg_duration_per_task_ms": 264292.92, "avg_output_tokens_per_task": 34254.08, "last_run_timestamp": "2026-10-31T13:21:05.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06290177, "avg_duration_per_task_ms": 485845.77, "avg_output_tokens_per_task": 14483.09, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.73, "accuracy_std_dev": 0.0268328157299975, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.074570136, "avg_duration_per_task_ms": 246781.984, "avg_output_tokens_per_task": 30037.412, "last_run_timestamp": "2026-10-31T11:52:51.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12856867000000002, "avg_duration_per_task_ms": 121477.86, "avg_output_tokens_per_task": 19799.68, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.24468226999999998, "avg_duration_per_task_ms": 158524.06, "avg_output_tokens_per_task": 22552.35, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Sail Research", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.22321662, "avg_duration_per_task_ms": 277142.38, "avg_output_tokens_per_task": 10411.9, "last_run_timestamp": "2026-11-06T22:14:35.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Modal", "accuracy": 0.73, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14279748, "avg_duration_per_task_ms": 129435.32, "avg_output_tokens_per_task": 7981.12, "last_run_timestamp": "2026-11-04T21:11:46.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7299998750000001, "accuracy_std_dev": 0.03299833750144659, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.15967390250000002, "avg_duration_per_task_ms": 222567.4375, "avg_output_tokens_per_task": 9532.0625, "last_run_timestamp": "2026-10-31T19:57:59.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7299997500000001, "accuracy_std_dev": 0.017320556188745798, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.32059346000000005, "avg_duration_per_task_ms": 156347.09, "avg_output_tokens_per_task": 21297.645, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.729654525083612, "accuracy_std_dev": 0.01804542249436126, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.18312612040133777, "avg_duration_per_task_ms": 543481.8829431438, "avg_output_tokens_per_task": 8776.65551839465, "last_run_timestamp": "2026-10-31T21:08:40.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7294444999999999, "accuracy_std_dev": 0.04038785494737249, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06920636333333333, "avg_duration_per_task_ms": 306647.6433333333, "avg_output_tokens_per_task": 12851.353333333333, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7293334, "accuracy_std_dev": 0.026533159130416407, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012403696, "avg_duration_per_task_ms": 92314.648, "avg_output_tokens_per_task": 10764.472, "last_run_timestamp": "2026-10-31T12:41:01.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7291665, "accuracy_std_dev": 0.023318429765959808, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0316769325, "avg_duration_per_task_ms": 89151.6375, "avg_output_tokens_per_task": 22436.51, "last_run_timestamp": "2026-10-31T11:37:01.556Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.7288890000000001, "accuracy_std_dev": 0.03142665378305492, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02386675333333333, "avg_duration_per_task_ms": 151228.52, "avg_output_tokens_per_task": 22554.02666666667, "last_run_timestamp": "2026-10-28T06:40:49.113Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.728889, "accuracy_std_dev": 0.01911599421078241, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017266966666666664, "avg_duration_per_task_ms": 122400.14, "avg_output_tokens_per_task": 11879.513333333334, "last_run_timestamp": "2026-10-31T11:33:22.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.728889, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04855964666666667, "avg_duration_per_task_ms": 115027.81333333334, "avg_output_tokens_per_task": 15070.966666666667, "last_run_timestamp": "2026-10-31T12:30:34.970Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.728889, "accuracy_std_dev": 0.03987630256458365, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01001504888888889, "avg_duration_per_task_ms": 391679.0466666667, "avg_output_tokens_per_task": 14314.308888888889, "last_run_timestamp": "2026-10-17T13:13:33.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7288886666666667, "accuracy_std_dev": 0.0318947884596981, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.035202753333333336, "avg_duration_per_task_ms": 333081.32666666666, "avg_output_tokens_per_task": 8294.333333333334, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.7288886666666667, "accuracy_std_dev": 0.017497738831695454, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033088113333333335, "avg_duration_per_task_ms": 134354.10666666666, "avg_output_tokens_per_task": 14586.926666666666, "last_run_timestamp": "2026-10-31T12:32:08.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.728735503448276, "accuracy_std_dev": 0.0506044752324886, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.07558873793103447, "avg_duration_per_task_ms": 486950.46206896554, "avg_output_tokens_per_task": 7447.337931034483, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7286996367713005, "accuracy_std_dev": 0.04475223154653922, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.03329973766816144, "avg_duration_per_task_ms": 1152472.0291479821, "avg_output_tokens_per_task": 52238.542600896864, "last_run_timestamp": "2026-10-31T20:03:31.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7285714285714285, "accuracy_std_dev": 0.035717562390258945, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07543138571428572, "avg_duration_per_task_ms": 221880.41142857142, "avg_output_tokens_per_task": 11178.914285714285, "last_run_timestamp": "2026-10-31T20:16:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.7283335, "accuracy_std_dev": 0.024626886091898288, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04604961333333333, "avg_duration_per_task_ms": 118630.39, "avg_output_tokens_per_task": 26239.556666666667, "last_run_timestamp": "2026-10-31T12:19:28.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.72833325, "accuracy_std_dev": 0.041197373838480274, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011627464999999998, "avg_duration_per_task_ms": 86463.365, "avg_output_tokens_per_task": 10669.235, "last_run_timestamp": "2026-10-31T12:31:32.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.728, "accuracy_std_dev": 0.02362657637492154, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.027497304, "avg_duration_per_task_ms": 93734.552, "avg_output_tokens_per_task": 12939.792, "last_run_timestamp": "2026-10-31T12:24:43.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7279999333333333, "accuracy_std_dev": 0.024092455251846428, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.024600684000000005, "avg_duration_per_task_ms": 183990.44, "avg_output_tokens_per_task": 23628.08, "last_run_timestamp": "2026-11-02T17:49:35.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7277778333333333, "accuracy_std_dev": 0.01651796253998121, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10709389666666666, "avg_duration_per_task_ms": 343686.82666666666, "avg_output_tokens_per_task": 14435.816666666668, "last_run_timestamp": "2026-10-17T19:16:53.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7274997500000001, "accuracy_std_dev": 0.030034653596928695, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04475519, "avg_duration_per_task_ms": 324824.455, "avg_output_tokens_per_task": 13203.02, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7273503282051282, "accuracy_std_dev": 0.04018278252249339, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.017554456410256408, "avg_duration_per_task_ms": 289835.1384615385, "avg_output_tokens_per_task": 6267.041025641026, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7271428571428571, "accuracy_std_dev": 0.01855313018202104, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.31224118, "avg_duration_per_task_ms": 126481.89142857143, "avg_output_tokens_per_task": 5238.86, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7271345697356426, "accuracy_std_dev": 0.02666397891606509, "run_count": 22, "total_questions": 1097, "avg_cost_per_task_usd": 0.4311616608933454, "avg_duration_per_task_ms": 251082.97538742024, "avg_output_tokens_per_task": 10876.747493163171, "last_run_timestamp": "2026-10-31T12:24:25.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7270409540816326, "accuracy_std_dev": 0.02754153279934688, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.04606436989795917, "avg_duration_per_task_ms": 81576.88520408163, "avg_output_tokens_per_task": 19891.04081632653, "last_run_timestamp": "2026-10-31T12:26:53.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Nebius", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07819403999999999, "avg_duration_per_task_ms": 528014.2, "avg_output_tokens_per_task": 10299.58, "last_run_timestamp": "2026-11-03T22:19:06.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.726667, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.24019618, "avg_duration_per_task_ms": 932058.35, "avg_output_tokens_per_task": 24816.04, "last_run_timestamp": "2026-10-31T11:58:06.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21341358, "avg_duration_per_task_ms": 101630.84, "avg_output_tokens_per_task": 8827.48, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.72666675, "accuracy_std_dev": 0.014907082584714534, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0051741124999999995, "avg_duration_per_task_ms": 172373.7025, "avg_output_tokens_per_task": 11668.895, "last_run_timestamp": "2026-11-06T22:18:15.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.72666675, "accuracy_std_dev": 0.02748728999351337, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.068011135, "avg_duration_per_task_ms": 209802.9225, "avg_output_tokens_per_task": 13044.05, "last_run_timestamp": "2026-10-31T19:39:42.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.023726809342082957, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06235516666666667, "avg_duration_per_task_ms": 501815.64666666667, "avg_output_tokens_per_task": 14648.713333333333, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.039752443827043155, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0691682688888889, "avg_duration_per_task_ms": 289187.1177777778, "avg_output_tokens_per_task": 12696.368888888888, "last_run_timestamp": "2026-10-31T19:48:30.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.30249026, "avg_duration_per_task_ms": 183822.13333333333, "avg_output_tokens_per_task": 10747.233333333334, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013071513333333333, "avg_duration_per_task_ms": 97597.57333333333, "avg_output_tokens_per_task": 4935.306666666666, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7266665833333333, "accuracy_std_dev": 0.02538595412315748, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07330062000000001, "avg_duration_per_task_ms": 226287.41166666665, "avg_output_tokens_per_task": 8532.536666666667, "last_run_timestamp": "2026-10-31T20:10:58.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7266665714285715, "accuracy_std_dev": 0.02225403120372413, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09929396571428571, "avg_duration_per_task_ms": 163639.28, "avg_output_tokens_per_task": 52902.33142857143, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018476000000000003, "avg_duration_per_task_ms": 104934.72, "avg_output_tokens_per_task": 10917.83, "last_run_timestamp": "2026-10-24T11:48:43.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.43167194999999997, "avg_duration_per_task_ms": 195075.98, "avg_output_tokens_per_task": 31329.14, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.7266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06725601, "avg_duration_per_task_ms": 640810.38, "avg_output_tokens_per_task": 15599.02, "last_run_timestamp": "2026-10-27T14:13:59.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.7266663333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024500659999999997, "avg_duration_per_task_ms": 72543.00666666667, "avg_output_tokens_per_task": 15175.193333333333, "last_run_timestamp": "2026-10-31T11:34:42.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.7264369310344827, "accuracy_std_dev": 0.006057869592521776, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.012712737931034484, "avg_duration_per_task_ms": 160320.86896551723, "avg_output_tokens_per_task": 8003.303448275862, "last_run_timestamp": "2026-10-31T12:11:53.396Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.726330531092437, "accuracy_std_dev": 0.031188754641318872, "run_count": 12, "total_questions": 595, "avg_cost_per_task_usd": 0.07654581848739497, "avg_duration_per_task_ms": 189475.03361344538, "avg_output_tokens_per_task": 8239.426890756302, "last_run_timestamp": "2026-10-31T19:01:31.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7260605454545455, "accuracy_std_dev": 0.023519428697311804, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09003248909090909, "avg_duration_per_task_ms": 647311.3927272727, "avg_output_tokens_per_task": 9449.287272727273, "last_run_timestamp": "2026-10-31T19:40:42.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StepFun", "accuracy": 0.7255556666666666, "accuracy_std_dev": 0.016960557004087128, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012908883333333333, "avg_duration_per_task_ms": 53544.27, "avg_output_tokens_per_task": 10836.613333333333, "last_run_timestamp": "2026-11-02T17:45:53.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7255072173913043, "accuracy_std_dev": 0.03099958588076344, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.35199391739130437, "avg_duration_per_task_ms": 204615.2852173913, "avg_output_tokens_per_task": 10253.284347826087, "last_run_timestamp": "2026-10-31T12:42:19.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7254545454545455, "accuracy_std_dev": 0.030460198481307724, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.34895750909090917, "avg_duration_per_task_ms": 232559.38545454547, "avg_output_tokens_per_task": 10874.458181818181, "last_run_timestamp": "2026-10-31T12:28:19.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.725, "accuracy_std_dev": 0.017240198505237704, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01070377, "avg_duration_per_task_ms": 64833.36, "avg_output_tokens_per_task": 4228.4, "last_run_timestamp": "2026-11-03T02:02:55.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7248484545454545, "accuracy_std_dev": 0.03023008675404398, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.32869586, "avg_duration_per_task_ms": 207276.92, "avg_output_tokens_per_task": 10330.558181818182, "last_run_timestamp": "2026-10-31T12:49:56.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7244444444444444, "accuracy_std_dev": 0.02948105899692626, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027190664444444446, "avg_duration_per_task_ms": 241956.15333333332, "avg_output_tokens_per_task": 9273.511111111111, "last_run_timestamp": "2026-10-24T19:14:23.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7244444444444444, "accuracy_std_dev": 0.022662439694648512, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06561171777777777, "avg_duration_per_task_ms": 200024.29111111112, "avg_output_tokens_per_task": 9471.135555555555, "last_run_timestamp": "2026-10-31T20:17:42.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.006285236475706843, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018751046666666667, "avg_duration_per_task_ms": 183683.96666666667, "avg_output_tokens_per_task": 11588.8, "last_run_timestamp": "2026-10-31T11:59:31.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12152117999999999, "avg_duration_per_task_ms": 471173.63333333336, "avg_output_tokens_per_task": 54771.27333333333, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7241666250000001, "accuracy_std_dev": 0.024252670244415866, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09570978000000002, "avg_duration_per_task_ms": 206305.7525, "avg_output_tokens_per_task": 12767.7075, "last_run_timestamp": "2026-10-31T20:36:53.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.724, "accuracy_std_dev": 0.02351363670723862, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.26574487199999997, "avg_duration_per_task_ms": 177434.862, "avg_output_tokens_per_task": 11402.85, "last_run_timestamp": "2026-10-31T12:56:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.723360643442623, "accuracy_std_dev": 0.019132270021092652, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.0756424631147541, "avg_duration_per_task_ms": 76631.30327868853, "avg_output_tokens_per_task": 8337.151639344262, "last_run_timestamp": "2026-10-31T11:46:18.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7233335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11131322, "avg_duration_per_task_ms": 83286.18, "avg_output_tokens_per_task": 3736.91, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.723333375, "accuracy_std_dev": 0.026457591854406843, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.007071109999999999, "avg_duration_per_task_ms": 91771.1525, "avg_output_tokens_per_task": 9121.6525, "last_run_timestamp": "2026-11-04T21:12:04.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.723333375, "accuracy_std_dev": 0.029814290943847292, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.041186885, "avg_duration_per_task_ms": 602276.8525, "avg_output_tokens_per_task": 9143.275, "last_run_timestamp": "2026-10-31T19:41:08.087Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.723333375, "accuracy_std_dev": 0.04203188274077637, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10373765500000001, "avg_duration_per_task_ms": 200657.97, "avg_output_tokens_per_task": 11980.6125, "last_run_timestamp": "2026-10-31T20:14:27.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02918011, "avg_duration_per_task_ms": 177614.61, "avg_output_tokens_per_task": 35023.7, "last_run_timestamp": "2026-10-31T11:38:56.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33860155, "avg_duration_per_task_ms": 113561.71, "avg_output_tokens_per_task": 10711.4, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7232361605839416, "accuracy_std_dev": 0.08188802109119138, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.04147013868613139, "avg_duration_per_task_ms": 272167.5200729927, "avg_output_tokens_per_task": 24002.558394160584, "last_run_timestamp": "2026-10-31T19:30:45.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7230302727272727, "accuracy_std_dev": 0.036221982175797626, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0770231490909091, "avg_duration_per_task_ms": 156976.8818181818, "avg_output_tokens_per_task": 8259.927272727273, "last_run_timestamp": "2026-10-31T19:53:09.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.7228067157894738, "accuracy_std_dev": 0.002917999999999976, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.06760965263157895, "avg_duration_per_task_ms": 49831.26315789474, "avg_output_tokens_per_task": 5886.4315789473685, "last_run_timestamp": "2026-10-31T12:12:21.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.722650186440678, "accuracy_std_dev": 0.03806732610044826, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.024282243451463792, "avg_duration_per_task_ms": 277851.95069337444, "avg_output_tokens_per_task": 9180.11093990755, "last_run_timestamp": "2026-10-31T13:24:52.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7225, "accuracy_std_dev": 0.029615564602924586, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05079202000000001, "avg_duration_per_task_ms": 151914.885, "avg_output_tokens_per_task": 28118.625, "last_run_timestamp": "2026-10-31T12:23:57.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.03456997912768952, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011571553333333331, "avg_duration_per_task_ms": 541667.7866666666, "avg_output_tokens_per_task": 18235.293333333335, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03683, "avg_duration_per_task_ms": 122927.25333333333, "avg_output_tokens_per_task": 9303.18, "last_run_timestamp": "2026-10-31T12:49:19.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.722083375, "accuracy_std_dev": 0.03692844197206775, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.089901815, "avg_duration_per_task_ms": 164446.19, "avg_output_tokens_per_task": 10597.3975, "last_run_timestamp": "2026-10-31T19:56:54.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.722, "accuracy_std_dev": 0.019843879550128302, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06701492399999999, "avg_duration_per_task_ms": 78660.404, "avg_output_tokens_per_task": 8431.312, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7218181818181818, "accuracy_std_dev": 0.026680618940681487, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08085123454545454, "avg_duration_per_task_ms": 229489.58727272728, "avg_output_tokens_per_task": 8760.714545454546, "last_run_timestamp": "2026-10-31T20:37:11.918Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.72166675, "accuracy_std_dev": 0.012801898733683979, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024317345, "avg_duration_per_task_ms": 116922.04, "avg_output_tokens_per_task": 11395.83, "last_run_timestamp": "2026-10-31T12:46:14.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.7216665, "accuracy_std_dev": 0.031754396022440755, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.018527804999999998, "avg_duration_per_task_ms": 90190.985, "avg_output_tokens_per_task": 8919.29, "last_run_timestamp": "2026-11-03T01:55:17.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.7214610958904109, "accuracy_std_dev": 0.02944405326868039, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.01699455479452055, "avg_duration_per_task_ms": 840567.9794520548, "avg_output_tokens_per_task": 10799.780821917808, "last_run_timestamp": "2026-10-31T13:36:58.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.7213334, "accuracy_std_dev": 0.01146975927559073, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012899843999999999, "avg_duration_per_task_ms": 152142.784, "avg_output_tokens_per_task": 13063.108, "last_run_timestamp": "2026-10-31T13:10:04.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7213332, "accuracy_std_dev": 0.030811331249395918, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.033439392, "avg_duration_per_task_ms": 410344.808, "avg_output_tokens_per_task": 8978.336, "last_run_timestamp": "2026-10-31T21:01:10.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.721250125, "accuracy_std_dev": 0.025052106116240477, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.21457064250000002, "avg_duration_per_task_ms": 1269183.795, "avg_output_tokens_per_task": 14006.3875, "last_run_timestamp": "2026-10-31T19:47:23.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.720833375, "accuracy_std_dev": 0.014312870571076042, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05266061, "avg_duration_per_task_ms": 391408.44, "avg_output_tokens_per_task": 9353.0625, "last_run_timestamp": "2026-10-31T20:46:45.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7201178737373739, "accuracy_std_dev": 0.03208603854872086, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.0926496590909091, "avg_duration_per_task_ms": 92170.53535353535, "avg_output_tokens_per_task": 7432.570707070707, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.72, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.026063740000000002, "avg_duration_per_task_ms": 127597.92, "avg_output_tokens_per_task": 12968.38, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0.035694340251642157, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05599238666666667, "avg_duration_per_task_ms": 288072.32666666666, "avg_output_tokens_per_task": 8411.226666666667, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.72, "accuracy_std_dev": 0.01247214674785381, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08792842000000001, "avg_duration_per_task_ms": 120135.21, "avg_output_tokens_per_task": 16462.21, "last_run_timestamp": "2026-10-31T11:24:22.174Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "PrimeIntellect", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00807376, "avg_duration_per_task_ms": 275288.98, "avg_output_tokens_per_task": 19939.9, "last_run_timestamp": "2026-11-01T18:47:31.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.72, "accuracy_std_dev": 0.03375490985700519, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08901049818181819, "avg_duration_per_task_ms": 307122.0472727273, "avg_output_tokens_per_task": 9253.218181818182, "last_run_timestamp": "2026-10-31T19:57:59.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.7199996666666667, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.046407413333333335, "avg_duration_per_task_ms": 163709.00666666665, "avg_output_tokens_per_task": 8100.34, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7196295555555555, "accuracy_std_dev": 0.023540924295603994, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04364436, "avg_duration_per_task_ms": 154693.13555555555, "avg_output_tokens_per_task": 29598.053333333333, "last_run_timestamp": "2026-11-05T00:27:56.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.71944425, "accuracy_std_dev": 0.06232871962576722, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.27328554166666674, "avg_duration_per_task_ms": 424932.33166666667, "avg_output_tokens_per_task": 10565.703333333333, "last_run_timestamp": "2026-10-31T12:33:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7193939090909091, "accuracy_std_dev": 0.021547202760681436, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08196447454545454, "avg_duration_per_task_ms": 229606.52181818182, "avg_output_tokens_per_task": 9247.236363636364, "last_run_timestamp": "2026-10-31T20:11:34.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7193939090909091, "accuracy_std_dev": 0.026278368554779535, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.04867809818181818, "avg_duration_per_task_ms": 168623.93818181817, "avg_output_tokens_per_task": 9688.069090909092, "last_run_timestamp": "2026-10-31T20:01:58.593Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7191666250000001, "accuracy_std_dev": 0.02145742393052752, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0557140475, "avg_duration_per_task_ms": 330536.0525, "avg_output_tokens_per_task": 9856.7875, "last_run_timestamp": "2026-10-31T20:51:04.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7191665, "accuracy_std_dev": 0.035968585831945094, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09730926000000001, "avg_duration_per_task_ms": 222643.045, "avg_output_tokens_per_task": 10115.1975, "last_run_timestamp": "2026-10-31T20:09:58.111Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7183335, "accuracy_std_dev": 0.031402344215838394, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.114731285, "avg_duration_per_task_ms": 88444.29, "avg_output_tokens_per_task": 8961.775, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.71833325, "accuracy_std_dev": 0.023033749394909635, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.059744359999999996, "avg_duration_per_task_ms": 328747.2175, "avg_output_tokens_per_task": 9865.755, "last_run_timestamp": "2026-10-31T20:35:24.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.71833325, "accuracy_std_dev": 0.028037588857416033, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1325301525, "avg_duration_per_task_ms": 255309.835, "avg_output_tokens_per_task": 10427.27, "last_run_timestamp": "2026-10-31T20:31:35.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7182456315789474, "accuracy_std_dev": 0.026827730127829193, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.06829176842105264, "avg_duration_per_task_ms": 115296.11368421053, "avg_output_tokens_per_task": 8771.72947368421, "last_run_timestamp": "2026-10-31T19:54:14.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.717778, "accuracy_std_dev": 0.046930323977857506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.045013793333333336, "avg_duration_per_task_ms": 196458.52, "avg_output_tokens_per_task": 18662.633333333335, "last_run_timestamp": "2026-10-31T19:03:35.054Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.7177779999999999, "accuracy_std_dev": 0.03499564699597175, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.039637780000000004, "avg_duration_per_task_ms": 223451.97333333333, "avg_output_tokens_per_task": 45456.886666666665, "last_run_timestamp": "2026-10-31T11:34:19.759Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.717499875, "accuracy_std_dev": 0.033654675052054425, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.051500805000000004, "avg_duration_per_task_ms": 406565.7275, "avg_output_tokens_per_task": 11732.8625, "last_run_timestamp": "2026-10-31T20:58:51.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01519146, "avg_duration_per_task_ms": 819823.24, "avg_output_tokens_per_task": 11470.58, "last_run_timestamp": "2026-10-31T20:19:18.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7166669999999999, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1280142, "avg_duration_per_task_ms": 531647.04, "avg_output_tokens_per_task": 56970.52, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.71666675, "accuracy_std_dev": 0.02624654474759486, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.026279967499999998, "avg_duration_per_task_ms": 219262.365, "avg_output_tokens_per_task": 10080.3425, "last_run_timestamp": "2026-10-31T20:20:16.475Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.022047789999999998, "avg_duration_per_task_ms": 63248.07, "avg_output_tokens_per_task": 8992.32, "last_run_timestamp": "2026-10-31T12:43:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015074639999999999, "avg_duration_per_task_ms": 124785.23, "avg_output_tokens_per_task": 11839.88, "last_run_timestamp": "2026-10-31T12:28:39.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.71669186, "avg_duration_per_task_ms": 277432.68, "avg_output_tokens_per_task": 22919.22, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.02603405888927041, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13610847, "avg_duration_per_task_ms": 230555.42, "avg_output_tokens_per_task": 11694.86, "last_run_timestamp": "2026-10-31T19:46:31.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7159998000000001, "accuracy_std_dev": 0.03295793548085197, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.24450749400000002, "avg_duration_per_task_ms": 314057.314, "avg_output_tokens_per_task": 21575.564, "last_run_timestamp": "2026-10-31T12:46:35.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.71583325, "accuracy_std_dev": 0.03770315157831637, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0668666975, "avg_duration_per_task_ms": 253791.08, "avg_output_tokens_per_task": 27006.42, "last_run_timestamp": "2026-10-31T19:22:37.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7157778333333332, "accuracy_std_dev": 0.032271291757105434, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.116951148, "avg_duration_per_task_ms": 125547.258, "avg_output_tokens_per_task": 8555.182666666668, "last_run_timestamp": "2026-10-31T19:56:11.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.03189478845969807, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01325666, "avg_duration_per_task_ms": 86099.67333333334, "avg_output_tokens_per_task": 8966.793333333333, "last_run_timestamp": "2026-10-31T13:08:15.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.009582875, "avg_duration_per_task_ms": 164124.5, "avg_output_tokens_per_task": 18445.333333333332, "last_run_timestamp": "2026-10-10T11:44:02.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7148900152284264, "accuracy_std_dev": 0.09117406530359592, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.14045050253807106, "avg_duration_per_task_ms": 1841531.8324873096, "avg_output_tokens_per_task": 104496.4695431472, "last_run_timestamp": "2026-10-31T19:11:32.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7145454545454546, "accuracy_std_dev": 0.032388915902494195, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.046899645454545454, "avg_duration_per_task_ms": 102311.42727272728, "avg_output_tokens_per_task": 8424.598181818183, "last_run_timestamp": "2026-10-31T19:54:42.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7145453636363637, "accuracy_std_dev": 0.028258254765877216, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0781004418181818, "avg_duration_per_task_ms": 129072.70181818181, "avg_output_tokens_per_task": 9377.12, "last_run_timestamp": "2026-10-31T19:18:12.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05452906, "avg_duration_per_task_ms": 272872.18, "avg_output_tokens_per_task": 39719.44, "last_run_timestamp": "2026-10-13T20:01:29.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08007529, "avg_duration_per_task_ms": 140906.22, "avg_output_tokens_per_task": 42659.23, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Fireworks", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01617958, "avg_duration_per_task_ms": 141600.68, "avg_output_tokens_per_task": 14742.64, "last_run_timestamp": "2026-10-31T12:42:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7125000625, "accuracy_std_dev": 0.023790114515983167, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.0387893925, "avg_duration_per_task_ms": 225024.16875, "avg_output_tokens_per_task": 9749.72875, "last_run_timestamp": "2026-10-31T20:38:36.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.712381, "accuracy_std_dev": 0.016106167044245584, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11192505714285715, "avg_duration_per_task_ms": 96202.06, "avg_output_tokens_per_task": 3694.017142857143, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7122223333333333, "accuracy_std_dev": 0.04016943897071779, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07876202666666668, "avg_duration_per_task_ms": 110695.52666666667, "avg_output_tokens_per_task": 8630.058333333332, "last_run_timestamp": "2026-10-31T19:45:55.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.7117822371364653, "accuracy_std_dev": 0.056118924341169225, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.0462677941834452, "avg_duration_per_task_ms": 1033622.2237136465, "avg_output_tokens_per_task": 60076.49888143177, "last_run_timestamp": "2026-10-31T13:23:14.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.006285236475706792, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06754876666666666, "avg_duration_per_task_ms": 139695.35333333333, "avg_output_tokens_per_task": 33581.94, "last_run_timestamp": "2026-10-31T12:01:01.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7111111666666666, "accuracy_std_dev": 0.06618325102425, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10397005333333333, "avg_duration_per_task_ms": 211488.60333333333, "avg_output_tokens_per_task": 11044.173333333334, "last_run_timestamp": "2026-10-17T19:18:15.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7109090454545455, "accuracy_std_dev": 0.02554809684866649, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.0988174109090909, "avg_duration_per_task_ms": 108917.78818181818, "avg_output_tokens_per_task": 9669.932727272728, "last_run_timestamp": "2026-10-31T19:37:51.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.71, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015407820000000001, "avg_duration_per_task_ms": 62181.78, "avg_output_tokens_per_task": 11850.44, "last_run_timestamp": "2026-10-31T11:31:06.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0134254, "avg_duration_per_task_ms": 120168.52, "avg_output_tokens_per_task": 11670.58, "last_run_timestamp": "2026-10-31T12:25:39.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12724396999999998, "avg_duration_per_task_ms": 73218.95, "avg_output_tokens_per_task": 3826.45, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7096295555555556, "accuracy_std_dev": 0.018083757273747114, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.022423851111111108, "avg_duration_per_task_ms": 274816.1711111111, "avg_output_tokens_per_task": 10081.8, "last_run_timestamp": "2026-10-31T19:56:09.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7093333, "accuracy_std_dev": 0.07889230465267193, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.010360992, "avg_duration_per_task_ms": 240638.89, "avg_output_tokens_per_task": 9031.174, "last_run_timestamp": "2026-10-31T12:26:55.059Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.709166625, "accuracy_std_dev": 0.011989726507905632, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0270094325, "avg_duration_per_task_ms": 227764.9775, "avg_output_tokens_per_task": 10406.5575, "last_run_timestamp": "2026-10-31T19:47:12.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.709166625, "accuracy_std_dev": 0.04593912865667321, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0539720225, "avg_duration_per_task_ms": 239606.0525, "avg_output_tokens_per_task": 8198.2525, "last_run_timestamp": "2026-10-31T20:09:59.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7090909090909091, "accuracy_std_dev": 0.15487416012037558, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.062087180000000006, "avg_duration_per_task_ms": 147570.8818181818, "avg_output_tokens_per_task": 16917.33090909091, "last_run_timestamp": "2026-10-31T19:40:48.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.708889, "accuracy_std_dev": 0.0031423825355929936, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017471633333333333, "avg_duration_per_task_ms": 219758.70666666667, "avg_output_tokens_per_task": 16040.473333333333, "last_run_timestamp": "2026-10-31T12:30:16.930Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.7083331666666667, "accuracy_std_dev": 0.017400462957602263, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.25373078000000004, "avg_duration_per_task_ms": 176420.9, "avg_output_tokens_per_task": 10605.743333333334, "last_run_timestamp": "2026-10-31T19:48:27.256Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7080952857142857, "accuracy_std_dev": 0.122226283771085, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.23844771999999997, "avg_duration_per_task_ms": 1324378.657142857, "avg_output_tokens_per_task": 65425.98, "last_run_timestamp": "2026-10-10T19:16:51.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7074073333333334, "accuracy_std_dev": 0.018972248885616532, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09800756444444444, "avg_duration_per_task_ms": 168166.58444444445, "avg_output_tokens_per_task": 11095.48, "last_run_timestamp": "2026-10-17T19:53:39.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.44632813, "avg_duration_per_task_ms": 132811.5, "avg_output_tokens_per_task": 20668.24, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.71176192, "avg_duration_per_task_ms": 285827.28, "avg_output_tokens_per_task": 22895.76, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1042244, "avg_duration_per_task_ms": 226024.67, "avg_output_tokens_per_task": 3543.23, "last_run_timestamp": "2026-10-30T00:17:12.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.70666675, "accuracy_std_dev": 0.05477228618049002, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.031395105, "avg_duration_per_task_ms": 328094.585, "avg_output_tokens_per_task": 10267.135, "last_run_timestamp": "2026-10-31T20:10:49.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.70666675, "accuracy_std_dev": 0.027080004924067123, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.017216295, "avg_duration_per_task_ms": 228198.02, "avg_output_tokens_per_task": 15657.435, "last_run_timestamp": "2026-10-31T13:09:38.196Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015417799999999999, "avg_duration_per_task_ms": 75266.33333333333, "avg_output_tokens_per_task": 12018.02, "last_run_timestamp": "2026-10-31T11:20:39.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.01962587106064057, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08575882, "avg_duration_per_task_ms": 316120.8933333333, "avg_output_tokens_per_task": 32977.22666666667, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.7063038567335244, "accuracy_std_dev": 0.03586910330405172, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.02153851862464183, "avg_duration_per_task_ms": 628235.6618911175, "avg_output_tokens_per_task": 9341.048710601719, "last_run_timestamp": "2026-10-31T19:57:15.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7061111666666666, "accuracy_std_dev": 0.023602995992152234, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.023716863333333334, "avg_duration_per_task_ms": 111467.81333333334, "avg_output_tokens_per_task": 29595.083333333332, "last_run_timestamp": "2026-10-31T19:52:48.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7057145714285713, "accuracy_std_dev": 0.030327119046712443, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.012995502857142857, "avg_duration_per_task_ms": 102610.46857142857, "avg_output_tokens_per_task": 4898.9228571428575, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7055555833333333, "accuracy_std_dev": 0.03440877813014756, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.041761648333333325, "avg_duration_per_task_ms": 145144.42333333334, "avg_output_tokens_per_task": 8526.811666666666, "last_run_timestamp": "2026-10-31T20:18:13.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7055555, "accuracy_std_dev": 0.02979349502408202, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03607245166666667, "avg_duration_per_task_ms": 168553.58333333334, "avg_output_tokens_per_task": 9088.338333333333, "last_run_timestamp": "2026-10-31T20:27:27.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.70499975, "accuracy_std_dev": 0.011902485735656231, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13260128, "avg_duration_per_task_ms": 477326.39, "avg_output_tokens_per_task": 55745.89, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7047618571428571, "accuracy_std_dev": 0.03161536123210892, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08984225142857143, "avg_duration_per_task_ms": 859687.0971428572, "avg_output_tokens_per_task": 9438.171428571428, "last_run_timestamp": "2026-10-31T21:09:07.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7044443333333332, "accuracy_std_dev": 0.022662526857249517, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02059840666666667, "avg_duration_per_task_ms": 207682.6, "avg_output_tokens_per_task": 9037.553333333333, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7042425454545455, "accuracy_std_dev": 0.024662964506480832, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12288133272727272, "avg_duration_per_task_ms": 163283.9090909091, "avg_output_tokens_per_task": 11693.267272727273, "last_run_timestamp": "2026-10-31T19:55:04.254Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7042424545454545, "accuracy_std_dev": 0.0349627848858641, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10044568909090908, "avg_duration_per_task_ms": 91693.66545454545, "avg_output_tokens_per_task": 8696.44, "last_run_timestamp": "2026-10-31T19:33:25.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.70416675, "accuracy_std_dev": 0.03264911731100092, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07095855000000001, "avg_duration_per_task_ms": 325685.87, "avg_output_tokens_per_task": 10970.505, "last_run_timestamp": "2026-10-31T21:34:09.712Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.025508095447524113, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021704752, "avg_duration_per_task_ms": 119683.016, "avg_output_tokens_per_task": 11904.78, "last_run_timestamp": "2026-10-31T12:25:42.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7033335, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08534018, "avg_duration_per_task_ms": 347658.09, "avg_output_tokens_per_task": 32908.85, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.7030302727272727, "accuracy_std_dev": 0.04093488390134421, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11610903454545456, "avg_duration_per_task_ms": 126684.9490909091, "avg_output_tokens_per_task": 9547.561818181819, "last_run_timestamp": "2026-10-31T21:47:15.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.702963, "accuracy_std_dev": 0.02847753091278963, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.2806454911111111, "avg_duration_per_task_ms": 494767.4533333333, "avg_output_tokens_per_task": 9478.277777777777, "last_run_timestamp": "2026-10-31T22:14:51.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.021999169459070246, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10818771333333332, "avg_duration_per_task_ms": 79804.25333333333, "avg_output_tokens_per_task": 3542.24, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.046294969462734856, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13726572666666667, "avg_duration_per_task_ms": 333190.6066666667, "avg_output_tokens_per_task": 10328.673333333334, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7012120909090909, "accuracy_std_dev": 0.026102771387012606, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11443949090909093, "avg_duration_per_task_ms": 89100.04545454546, "avg_output_tokens_per_task": 9820.098181818183, "last_run_timestamp": "2026-10-31T19:57:32.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7008332500000001, "accuracy_std_dev": 0.04182460234344735, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06761195, "avg_duration_per_task_ms": 346627.2725, "avg_output_tokens_per_task": 10080.1925, "last_run_timestamp": "2026-10-31T20:44:35.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29769207999999997, "avg_duration_per_task_ms": 128375.6, "avg_output_tokens_per_task": 8500.86, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0355865, "avg_duration_per_task_ms": 154491.8, "avg_output_tokens_per_task": 12574.4, "last_run_timestamp": "2026-10-31T11:15:31.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0161093, "avg_duration_per_task_ms": 215051.72, "avg_output_tokens_per_task": 15584.08, "last_run_timestamp": "2026-10-31T12:24:55.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07376152, "avg_duration_per_task_ms": 750952.84, "avg_output_tokens_per_task": 11282.28, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": 0.004714280910170736, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.634261665, "avg_duration_per_task_ms": 673734.145, "avg_output_tokens_per_task": 52923.67, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05778106, "avg_duration_per_task_ms": 103456.94, "avg_output_tokens_per_task": 4693.4, "last_run_timestamp": "2026-10-28T00:06:02.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.6992855714285715, "accuracy_std_dev": 0.022473227207102254, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.1477709457142857, "avg_duration_per_task_ms": 78438.86571428571, "avg_output_tokens_per_task": 10556.58, "last_run_timestamp": "2026-10-24T19:06:47.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.6987950361445784, "accuracy_std_dev": 0.0178749221984321, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.04930324899598394, "avg_duration_per_task_ms": 404321.7309236948, "avg_output_tokens_per_task": 8360.08032128514, "last_run_timestamp": "2026-10-31T21:01:44.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.6983862420701168, "accuracy_std_dev": 0.027878199943260105, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.07633183305509182, "avg_duration_per_task_ms": 154743.70951585978, "avg_output_tokens_per_task": 9542.741235392321, "last_run_timestamp": "2026-10-31T19:41:05.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.698333375, "accuracy_std_dev": 0.030867217869195378, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05358634250000001, "avg_duration_per_task_ms": 348281.08, "avg_output_tokens_per_task": 9167.5225, "last_run_timestamp": "2026-10-31T22:10:13.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.698333375, "accuracy_std_dev": 0.043301188398638384, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06877143999999999, "avg_duration_per_task_ms": 73336.0625, "avg_output_tokens_per_task": 9928.085, "last_run_timestamp": "2026-10-31T19:23:52.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.698333, "accuracy_std_dev": 0.0050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05153802, "avg_duration_per_task_ms": 227792.72, "avg_output_tokens_per_task": 6678.6, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.6977779999999999, "accuracy_std_dev": 0.03095182238038119, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.051447179999999995, "avg_duration_per_task_ms": 808105.6666666666, "avg_output_tokens_per_task": 8854.186666666666, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013071946666666667, "avg_duration_per_task_ms": 88036.78, "avg_output_tokens_per_task": 11978.486666666666, "last_run_timestamp": "2026-10-31T11:55:17.583Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14221055333333335, "avg_duration_per_task_ms": 130884.03333333334, "avg_output_tokens_per_task": 4996.426666666666, "last_run_timestamp": "2026-10-30T00:14:00.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6975001249999999, "accuracy_std_dev": 0.025806460486850466, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.045653685, "avg_duration_per_task_ms": 547622.5375, "avg_output_tokens_per_task": 9024.185, "last_run_timestamp": "2026-10-31T20:03:54.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.6969061874251495, "accuracy_std_dev": 0.026180217814685852, "run_count": 34, "total_questions": 1670, "avg_cost_per_task_usd": 0.10252084191616766, "avg_duration_per_task_ms": 73055.79640718563, "avg_output_tokens_per_task": 8098.897005988024, "last_run_timestamp": "2026-10-31T20:07:10.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.696667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01321477, "avg_duration_per_task_ms": 116452.62, "avg_output_tokens_per_task": 5139.02, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0061224999999999995, "avg_duration_per_task_ms": 182716.13, "avg_output_tokens_per_task": 9606.88, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.6963472739726028, "accuracy_std_dev": 0.05315590618172168, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.01628368493150685, "avg_duration_per_task_ms": 275676.23287671234, "avg_output_tokens_per_task": 12531.828767123288, "last_run_timestamp": "2026-10-31T13:29:36.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.6957762602739725, "accuracy_std_dev": 0.0313433490574777, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.011891609589041095, "avg_duration_per_task_ms": 271916.02397260274, "avg_output_tokens_per_task": 12921.842465753425, "last_run_timestamp": "2026-11-04T21:26:44.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.6955782448979592, "accuracy_std_dev": 0.13306624277967724, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.07090773979591837, "avg_duration_per_task_ms": 158871.75, "avg_output_tokens_per_task": 16373.045918367347, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6955556666666667, "accuracy_std_dev": 0.021998731726069, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04300826666666667, "avg_duration_per_task_ms": 173663.04666666666, "avg_output_tokens_per_task": 8296.866666666667, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.6948719487179488, "accuracy_std_dev": 0.16500680809647672, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.024616958974358973, "avg_duration_per_task_ms": 297869.91282051284, "avg_output_tokens_per_task": 9525.128205128205, "last_run_timestamp": "2026-10-31T19:23:32.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.694073888888889, "accuracy_std_dev": 0.014209898212509368, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0217813, "avg_duration_per_task_ms": 480864.6, "avg_output_tokens_per_task": 18593.075555555555, "last_run_timestamp": "2026-10-31T19:50:01.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.6940476428571428, "accuracy_std_dev": 0.033242348446365685, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.12397131285714287, "avg_duration_per_task_ms": 129105.92571428571, "avg_output_tokens_per_task": 8940.347142857143, "last_run_timestamp": "2026-10-31T20:01:39.270Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.018257236010689022, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08853248000000001, "avg_duration_per_task_ms": 338024.125, "avg_output_tokens_per_task": 32557.16, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.13946158, "avg_duration_per_task_ms": 819620.14, "avg_output_tokens_per_task": 28549.28, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02754302, "avg_duration_per_task_ms": 209408.8, "avg_output_tokens_per_task": 14914.54, "last_run_timestamp": "2026-10-24T12:03:24.771Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6928572857142857, "accuracy_std_dev": 0.07897025990969059, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.047876402857142855, "avg_duration_per_task_ms": 481015.8685714286, "avg_output_tokens_per_task": 15710.031428571428, "last_run_timestamp": "2026-10-31T19:37:26.520Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6923937315436242, "accuracy_std_dev": 0.032633747756987186, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.01647934899328859, "avg_duration_per_task_ms": 211946.26845637584, "avg_output_tokens_per_task": 8844.79865771812, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.6919048571428571, "accuracy_std_dev": 0.1628112211782438, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01966778285714286, "avg_duration_per_task_ms": 333360.0657142857, "avg_output_tokens_per_task": 11239.56, "last_run_timestamp": "2026-11-04T18:04:38.320Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6918933673469387, "accuracy_std_dev": 0.11325810487735402, "run_count": 12, "total_questions": 588, "avg_cost_per_task_usd": 0.3106834897959183, "avg_duration_per_task_ms": 1590211.7602040817, "avg_output_tokens_per_task": 51719.284013605444, "last_run_timestamp": "2026-10-31T12:54:39.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.69166675, "accuracy_std_dev": 0.09607597832542482, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09531616, "avg_duration_per_task_ms": 242770.45, "avg_output_tokens_per_task": 22128.1325, "last_run_timestamp": "2026-10-31T19:46:05.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.6909721458333333, "accuracy_std_dev": 0.0024635000000000074, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.0246803125, "avg_duration_per_task_ms": 48638.177083333336, "avg_output_tokens_per_task": 11672.458333333334, "last_run_timestamp": "2026-10-31T11:12:51.723Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.69, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10882898999999999, "avg_duration_per_task_ms": 135482.83, "avg_output_tokens_per_task": 51335.48, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01750064, "avg_duration_per_task_ms": 190406.46, "avg_output_tokens_per_task": 15714.76, "last_run_timestamp": "2026-10-31T12:31:30.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "PrimeIntellect", "accuracy": 0.69, "accuracy_std_dev": 0.016667000000000043, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01877389, "avg_duration_per_task_ms": 125629.58, "avg_output_tokens_per_task": 15732.46, "last_run_timestamp": "2026-10-31T11:20:36.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.69, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07366, "avg_duration_per_task_ms": 63569.01, "avg_output_tokens_per_task": 3649.5, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6873332000000001, "accuracy_std_dev": 0.026686126789526168, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.10512817333333335, "avg_duration_per_task_ms": 76744.592, "avg_output_tokens_per_task": 9600.512666666667, "last_run_timestamp": "2026-11-02T21:38:02.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6866668333333333, "accuracy_std_dev": 0.029313263337134965, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03567404666666667, "avg_duration_per_task_ms": 155529.32333333333, "avg_output_tokens_per_task": 9866.113333333333, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.04496909432599238, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0149933775, "avg_duration_per_task_ms": 333684.6375, "avg_output_tokens_per_task": 19970.01, "last_run_timestamp": "2026-10-31T20:18:22.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01561806, "avg_duration_per_task_ms": 117250.14, "avg_output_tokens_per_task": 14519.81, "last_run_timestamp": "2026-10-24T12:06:50.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.6844443333333333, "accuracy_std_dev": 0.016178098288186456, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06359924333333333, "avg_duration_per_task_ms": 60968.18666666667, "avg_output_tokens_per_task": 4743.64, "last_run_timestamp": "2026-11-02T17:36:19.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6841667499999999, "accuracy_std_dev": 0.019982613972838976, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.00873134, "avg_duration_per_task_ms": 302344.745, "avg_output_tokens_per_task": 8518.3375, "last_run_timestamp": "2026-11-03T03:04:12.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.6838888333333334, "accuracy_std_dev": 0.04670285039094391, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09925613333333333, "avg_duration_per_task_ms": 184779.46, "avg_output_tokens_per_task": 7335.005, "last_run_timestamp": "2026-10-31T19:10:43.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.6835019898989899, "accuracy_std_dev": 0.023980399500164742, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.03288487878787879, "avg_duration_per_task_ms": 455208.9646464646, "avg_output_tokens_per_task": 20918.757575757576, "last_run_timestamp": "2026-11-05T15:52:10.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.6833334999999999, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018626550000000002, "avg_duration_per_task_ms": 239747.89, "avg_output_tokens_per_task": 15949.04, "last_run_timestamp": "2026-10-31T19:36:36.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.6830808080808081, "accuracy_std_dev": 0.024660860760271505, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.020343022727272727, "avg_duration_per_task_ms": 329622.595959596, "avg_output_tokens_per_task": 16902.575757575756, "last_run_timestamp": "2026-11-04T19:25:46.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.682995527027027, "accuracy_std_dev": 0.017120226436067428, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.03385346621621622, "avg_duration_per_task_ms": 512642.8918918919, "avg_output_tokens_per_task": 19026.391891891893, "last_run_timestamp": "2026-10-17T19:21:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.021998563367234276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.014046139999999999, "avg_duration_per_task_ms": 71689.16, "avg_output_tokens_per_task": 5022.28, "last_run_timestamp": "2026-10-30T00:15:40.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.68222225, "accuracy_std_dev": 0.06499753651629805, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.108739995, "avg_duration_per_task_ms": 80556.50333333333, "avg_output_tokens_per_task": 8974.771666666667, "last_run_timestamp": "2026-10-31T19:47:27.755Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6816577724867724, "accuracy_std_dev": 0.02279793656232948, "run_count": 4, "total_questions": 189, "avg_cost_per_task_usd": 0.13611332804232804, "avg_duration_per_task_ms": 393574.9259259259, "avg_output_tokens_per_task": 10439.26455026455, "last_run_timestamp": "2026-10-17T19:51:53.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.68125, "accuracy_std_dev": 0.0202715728731147, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0174918175, "avg_duration_per_task_ms": 123091.5525, "avg_output_tokens_per_task": 21340.095, "last_run_timestamp": "2026-10-31T11:46:38.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.6800001111111111, "accuracy_std_dev": 0.0689962496749633, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09411114000000001, "avg_duration_per_task_ms": 131955.26888888888, "avg_output_tokens_per_task": 8809.797777777778, "last_run_timestamp": "2026-10-31T19:19:57.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6799998571428572, "accuracy_std_dev": 0.0198405229080613, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.015568199999999999, "avg_duration_per_task_ms": 259045.84285714285, "avg_output_tokens_per_task": 18504.7, "last_run_timestamp": "2026-10-31T20:09:24.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.6787879090909091, "accuracy_std_dev": 0.033492690704511906, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.016563767272727275, "avg_duration_per_task_ms": 290847.2818181818, "avg_output_tokens_per_task": 19763.932727272728, "last_run_timestamp": "2026-11-01T13:16:57.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.67833325, "accuracy_std_dev": 0.019075602511781895, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01068972, "avg_duration_per_task_ms": 147233.725, "avg_output_tokens_per_task": 15363.5, "last_run_timestamp": "2026-11-04T18:22:16.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.678333, "accuracy_std_dev": 0.0050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0069528, "avg_duration_per_task_ms": 471037.61, "avg_output_tokens_per_task": 19230.47, "last_run_timestamp": "2026-10-31T13:27:50.961Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.017497908167041606, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06315626, "avg_duration_per_task_ms": 88978.8, "avg_output_tokens_per_task": 4762.613333333334, "last_run_timestamp": "2026-11-03T15:08:23.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6763826771300447, "accuracy_std_dev": 0.027382172972416883, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.021115630044843046, "avg_duration_per_task_ms": 298545.9484304933, "avg_output_tokens_per_task": 16751.952914798207, "last_run_timestamp": "2026-10-31T20:14:20.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.67583325, "accuracy_std_dev": 0.10893112439145894, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0162695775, "avg_duration_per_task_ms": 256380.655, "avg_output_tokens_per_task": 10959.865, "last_run_timestamp": "2026-10-31T19:58:02.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.00831485358994507, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.014157986666666667, "avg_duration_per_task_ms": 117427.1, "avg_output_tokens_per_task": 13587.66, "last_run_timestamp": "2026-10-31T11:44:53.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.63842362, "avg_duration_per_task_ms": 88201.35333333333, "avg_output_tokens_per_task": 3898.48, "last_run_timestamp": "2026-10-30T00:17:00.667Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.6736109999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.11439916666666666, "avg_duration_per_task_ms": 982669.25, "avg_output_tokens_per_task": 30745.75, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6733780939597316, "accuracy_std_dev": 0.01094232519470459, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.006236456375838926, "avg_duration_per_task_ms": 189635.3489932886, "avg_output_tokens_per_task": 9352.58389261745, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.6292479799999999, "avg_duration_per_task_ms": 729264.79, "avg_output_tokens_per_task": 54931.75, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.007971766666666666, "avg_duration_per_task_ms": 205309.70666666667, "avg_output_tokens_per_task": 9557.78, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.6733330000000001, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.5819717300000002, "avg_duration_per_task_ms": 314318.37, "avg_output_tokens_per_task": 23725.55, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6716665, "accuracy_std_dev": 0.01443369899748499, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.014968054999999997, "avg_duration_per_task_ms": 194795.94, "avg_output_tokens_per_task": 10086.53, "last_run_timestamp": "2026-10-13T22:01:15.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.67, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.022004160000000002, "avg_duration_per_task_ms": 232974.22, "avg_output_tokens_per_task": 9887.82, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.6690909090909091, "accuracy_std_dev": 0.14703195632270777, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12102286727272728, "avg_duration_per_task_ms": 359186.4090909091, "avg_output_tokens_per_task": 8810.841818181818, "last_run_timestamp": "2026-10-31T19:48:58.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6688886666666667, "accuracy_std_dev": 0.006285236475706792, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028048506666666667, "avg_duration_per_task_ms": 240999.82, "avg_output_tokens_per_task": 10014.44, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0755104, "avg_duration_per_task_ms": 77993.72, "avg_output_tokens_per_task": 14223.7, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.6666666666666666, "accuracy_std_dev": 0.0524933858267454, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03491572666666667, "avg_duration_per_task_ms": 274075.64666666667, "avg_output_tokens_per_task": 23860.97333333333, "last_run_timestamp": "2026-10-31T19:43:40.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.013174700000000001, "avg_duration_per_task_ms": 181950.26, "avg_output_tokens_per_task": 13269.31, "last_run_timestamp": "2026-10-31T12:42:39.997Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08889954, "avg_duration_per_task_ms": 108433.06, "avg_output_tokens_per_task": 12861.73, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.66583325, "accuracy_std_dev": 0.03639120582211448, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.006095729999999999, "avg_duration_per_task_ms": 325369.79, "avg_output_tokens_per_task": 18643.27, "last_run_timestamp": "2026-10-31T19:46:03.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6644446666666667, "accuracy_std_dev": 0.02997950740459595, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.022058193333333333, "avg_duration_per_task_ms": 220747.36, "avg_output_tokens_per_task": 10146.033333333333, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6643136764705884, "accuracy_std_dev": 0.02945620942699677, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.12474326823529412, "avg_duration_per_task_ms": 118894.24411764706, "avg_output_tokens_per_task": 12635.786470588235, "last_run_timestamp": "2026-10-31T12:26:00.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6622223333333334, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008384560000000001, "avg_duration_per_task_ms": 353358.7133333333, "avg_output_tokens_per_task": 13464.853333333333, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6618180909090909, "accuracy_std_dev": 0.03062851381683575, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11598973272727274, "avg_duration_per_task_ms": 123395.72363636363, "avg_output_tokens_per_task": 12658.365454545454, "last_run_timestamp": "2026-10-31T12:42:18.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6613334000000001, "accuracy_std_dev": 0.01758814752724117, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10186603200000001, "avg_duration_per_task_ms": 156255.32, "avg_output_tokens_per_task": 13208.728, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6604760000000001, "accuracy_std_dev": 0.025350253033394797, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008717225714285714, "avg_duration_per_task_ms": 574345.4257142857, "avg_output_tokens_per_task": 17634.762857142858, "last_run_timestamp": "2026-10-31T20:07:05.983Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6603702222222223, "accuracy_std_dev": 0.014353851715192269, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02473104222222222, "avg_duration_per_task_ms": 151337.52666666667, "avg_output_tokens_per_task": 16990.524444444443, "last_run_timestamp": "2026-11-05T00:31:57.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.66, "accuracy_std_dev": 0.005443582705045132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029202206666666668, "avg_duration_per_task_ms": 237856.5, "avg_output_tokens_per_task": 10714.64, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06369964, "avg_duration_per_task_ms": 706011.28, "avg_output_tokens_per_task": 8805.34, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.6577835126903554, "accuracy_std_dev": 0.125359473887098, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.02656375888324873, "avg_duration_per_task_ms": 1104661.6015228427, "avg_output_tokens_per_task": 34353.058375634515, "last_run_timestamp": "2026-10-17T21:28:32.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.6577776666666667, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.022382520000000003, "avg_duration_per_task_ms": 235251.81333333332, "avg_output_tokens_per_task": 9436.286666666667, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6575758181818182, "accuracy_std_dev": 0.02640359841460382, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11578603454545454, "avg_duration_per_task_ms": 129057.64181818182, "avg_output_tokens_per_task": 12484.370909090909, "last_run_timestamp": "2026-10-31T11:44:30.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6575, "accuracy_std_dev": 0.0675721425182005, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.056107382500000004, "avg_duration_per_task_ms": 767450.61, "avg_output_tokens_per_task": 18037.37, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6562239298245615, "accuracy_std_dev": 0.018946289616967236, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.01692722807017544, "avg_duration_per_task_ms": 312804.7944862155, "avg_output_tokens_per_task": 22811.754385964912, "last_run_timestamp": "2026-10-31T12:28:49.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.656161606060606, "accuracy_std_dev": 0.026312321205253775, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.12037706606060608, "avg_duration_per_task_ms": 121292.18242424242, "avg_output_tokens_per_task": 12576.998787878787, "last_run_timestamp": "2026-10-31T12:17:41.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.65583325, "accuracy_std_dev": 0.09059597044674504, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04999349, "avg_duration_per_task_ms": 312377.32, "avg_output_tokens_per_task": 7667.325, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6557823102040816, "accuracy_std_dev": 0.009318408855593308, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.014797057142857142, "avg_duration_per_task_ms": 277907.2775510204, "avg_output_tokens_per_task": 20245.579591836733, "last_run_timestamp": "2026-10-10T11:18:54.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.03807065880409451, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021039175555555557, "avg_duration_per_task_ms": 248791.5977777778, "avg_output_tokens_per_task": 26269.908888888887, "last_run_timestamp": "2026-10-31T12:25:21.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.012570472951413636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.030086579999999998, "avg_duration_per_task_ms": 125366.48, "avg_output_tokens_per_task": 16133.046666666667, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.65541675, "accuracy_std_dev": 0.02332950117474654, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.0099308575, "avg_duration_per_task_ms": 308960.4, "avg_output_tokens_per_task": 17384.11375, "last_run_timestamp": "2026-10-31T19:05:31.409Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.6546664, "accuracy_std_dev": 0.03512514515614133, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.01491196, "avg_duration_per_task_ms": 86567.944, "avg_output_tokens_per_task": 19737.508, "last_run_timestamp": "2026-10-31T19:08:06.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.65416675, "accuracy_std_dev": 0.03373725891262509, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.010956202499999998, "avg_duration_per_task_ms": 539774.1775, "avg_output_tokens_per_task": 18259.87, "last_run_timestamp": "2026-10-31T19:41:05.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6535409120879121, "accuracy_std_dev": 0.03689102528724027, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.012232007326007327, "avg_duration_per_task_ms": 1066692.706959707, "avg_output_tokens_per_task": 22783.287545787545, "last_run_timestamp": "2026-11-04T19:03:50.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Sail Research", "accuracy": 0.6533335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01241216, "avg_duration_per_task_ms": 170776.85, "avg_output_tokens_per_task": 17557.16, "last_run_timestamp": "2026-11-06T22:04:56.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6533334545454546, "accuracy_std_dev": 0.023268429213552766, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1143488090909091, "avg_duration_per_task_ms": 136176.96727272728, "avg_output_tokens_per_task": 12485.16, "last_run_timestamp": "2026-10-31T11:42:13.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2856746933333333, "avg_duration_per_task_ms": 183725.1, "avg_output_tokens_per_task": 8628.813333333334, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6518188195991091, "accuracy_std_dev": 0.02538535107955173, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.012973, "avg_duration_per_task_ms": 774691.5612472161, "avg_output_tokens_per_task": 26354.719376391982, "last_run_timestamp": "2026-10-31T19:22:53.448Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.65166675, "accuracy_std_dev": 0.028819327042585528, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01676374, "avg_duration_per_task_ms": 406825.29, "avg_output_tokens_per_task": 27732.185, "last_run_timestamp": "2026-10-31T19:20:36.815Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.65029765625, "accuracy_std_dev": 0.07423887380727727, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.025986263392857145, "avg_duration_per_task_ms": 280869.27901785716, "avg_output_tokens_per_task": 24958.395089285714, "last_run_timestamp": "2026-10-17T19:25:29.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6488889999999999, "accuracy_std_dev": 0.019115994210782406, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08501352666666667, "avg_duration_per_task_ms": 97755.09333333334, "avg_output_tokens_per_task": 12128.533333333333, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.6480655267857143, "accuracy_std_dev": 0.12510981874337443, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.04457617410714286, "avg_duration_per_task_ms": 1953706.560267857, "avg_output_tokens_per_task": 78316.31919642857, "last_run_timestamp": "2026-10-31T11:55:35.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6479167499999999, "accuracy_std_dev": 0.03374491834243344, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0332404675, "avg_duration_per_task_ms": 56284.1875, "avg_output_tokens_per_task": 14744.0125, "last_run_timestamp": "2026-10-31T18:54:40.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6466665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0295618, "avg_duration_per_task_ms": 121397.75, "avg_output_tokens_per_task": 15953.93, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6421009059720457, "accuracy_std_dev": 0.04038730565548258, "run_count": 16, "total_questions": 787, "avg_cost_per_task_usd": 0.0215750292249047, "avg_duration_per_task_ms": 328009.2465057179, "avg_output_tokens_per_task": 17335.555273189326, "last_run_timestamp": "2026-10-31T20:19:17.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6413336000000001, "accuracy_std_dev": 0.014847377621654282, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.03101342, "avg_duration_per_task_ms": 232783.988, "avg_output_tokens_per_task": 15818.512, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.6355553333333334, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008414553333333331, "avg_duration_per_task_ms": 1275373.5733333332, "avg_output_tokens_per_task": 17754.266666666666, "last_run_timestamp": "2026-10-31T21:55:54.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.634583375, "accuracy_std_dev": 0.061732736285008256, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.024563202499999996, "avg_duration_per_task_ms": 408597.23, "avg_output_tokens_per_task": 15674.2925, "last_run_timestamp": "2026-10-31T19:07:04.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6333333333333333, "accuracy_std_dev": 0.019625871060640607, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02406874, "avg_duration_per_task_ms": 135941.56666666668, "avg_output_tokens_per_task": 10001.226666666667, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6331658693467337, "accuracy_std_dev": 0.03188449882289982, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.03419022110552764, "avg_duration_per_task_ms": 256833.6432160804, "avg_output_tokens_per_task": 16666.94974874372, "last_run_timestamp": "2026-10-31T19:04:42.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6326291619718309, "accuracy_std_dev": 0.04710741386094833, "run_count": 3, "total_questions": 142, "avg_cost_per_task_usd": 0.022965091549295776, "avg_duration_per_task_ms": 52073.43661971831, "avg_output_tokens_per_task": 11681.894366197183, "last_run_timestamp": "2026-10-24T18:40:18.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.6322223333333333, "accuracy_std_dev": 0.13740533890848478, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05561755666666665, "avg_duration_per_task_ms": 502939.72, "avg_output_tokens_per_task": 9552.823333333334, "last_run_timestamp": "2026-10-31T19:32:40.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.63083325, "accuracy_std_dev": 0.12670763646654254, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.051196177499999995, "avg_duration_per_task_ms": 879758.5, "avg_output_tokens_per_task": 27053.105, "last_run_timestamp": "2026-10-31T19:22:14.979Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6282699518987342, "accuracy_std_dev": 0.030128786590858582, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.040355772151898735, "avg_duration_per_task_ms": 83816.96455696202, "avg_output_tokens_per_task": 19846.754430379748, "last_run_timestamp": "2026-10-31T18:59:19.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.0356942779945968, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021180666666666667, "avg_duration_per_task_ms": 241944.27333333335, "avg_output_tokens_per_task": 9711.62, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.14676766724392065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03268331333333334, "avg_duration_per_task_ms": 135271.32666666666, "avg_output_tokens_per_task": 12787.166666666666, "last_run_timestamp": "2026-10-31T11:51:01.680Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.6225, "accuracy_std_dev": 0.024706510614613316, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.068399865, "avg_duration_per_task_ms": 32162.2675, "avg_output_tokens_per_task": 21719.0175, "last_run_timestamp": "2026-10-31T19:45:42.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.62, "accuracy_std_dev": 0.02177351432359967, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028176926666666664, "avg_duration_per_task_ms": 213805.89333333334, "avg_output_tokens_per_task": 9686.44, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "PrimeIntellect", "accuracy": 0.62, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006431559999999999, "avg_duration_per_task_ms": 321502.32, "avg_output_tokens_per_task": 15491.98, "last_run_timestamp": "2026-11-01T06:36:42.081Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.6166665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21128705, "avg_duration_per_task_ms": 934328.99, "avg_output_tokens_per_task": 27359.88, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6166665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12254571, "avg_duration_per_task_ms": 607769.66, "avg_output_tokens_per_task": 27082.3, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6133335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12268729, "avg_duration_per_task_ms": 618681.49, "avg_output_tokens_per_task": 27773.81, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6076188571428572, "accuracy_std_dev": 0.03274303997331158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.046068719999999994, "avg_duration_per_task_ms": 191059.44, "avg_output_tokens_per_task": 3405.154285714286, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.6073332, "accuracy_std_dev": 0.020912477934477324, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0484729, "avg_duration_per_task_ms": 95144.364, "avg_output_tokens_per_task": 4713.688, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.605, "accuracy_std_dev": 0.02598076211353318, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.047004014999999996, "avg_duration_per_task_ms": 99148.785, "avg_output_tokens_per_task": 4762.08, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6000001666666667, "accuracy_std_dev": 0.03771239112642186, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19008634000000002, "avg_duration_per_task_ms": 186975.22, "avg_output_tokens_per_task": 11452.75, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.59966491959799, "accuracy_std_dev": 0.08352485507909829, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.028319288944723613, "avg_duration_per_task_ms": 1740003.0175879397, "avg_output_tokens_per_task": 55612.85427135678, "last_run_timestamp": "2026-10-31T19:40:44.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.5975, "accuracy_std_dev": 0.036959023857510084, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09500824249999999, "avg_duration_per_task_ms": 57134.1525, "avg_output_tokens_per_task": 3617.775, "last_run_timestamp": "2026-10-31T19:06:25.403Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.591111, "accuracy_std_dev": 0.006285295405415193, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029060479999999996, "avg_duration_per_task_ms": 77755.87, "avg_output_tokens_per_task": 7763.98, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.58916675, "accuracy_std_dev": 0.017058058540979977, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.020928054999999997, "avg_duration_per_task_ms": 95089.825, "avg_output_tokens_per_task": 4516.935, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.5833335, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14855325, "avg_duration_per_task_ms": 501107.83, "avg_output_tokens_per_task": 23998.36, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.58, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20638665, "avg_duration_per_task_ms": 179102.02, "avg_output_tokens_per_task": 12109.41, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5799547837837838, "accuracy_std_dev": 0.055903572770937965, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.19823583783783785, "avg_duration_per_task_ms": 2181766.8243243243, "avg_output_tokens_per_task": 43743.00675675676, "last_run_timestamp": "2026-10-17T19:42:10.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.575129585492228, "accuracy_std_dev": 0.04407725366331256, "run_count": 4, "total_questions": 193, "avg_cost_per_task_usd": 0.06909092746113989, "avg_duration_per_task_ms": 82524.34196891192, "avg_output_tokens_per_task": 4092.2538860103627, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.5666665000000001, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028862480000000003, "avg_duration_per_task_ms": 75034.33, "avg_output_tokens_per_task": 7701.06, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.55, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.2373302, "avg_duration_per_task_ms": 164429.2, "avg_output_tokens_per_task": 12508.34, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.546667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03543453, "avg_duration_per_task_ms": 61133.82, "avg_output_tokens_per_task": 6656.99, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.545139, "accuracy_std_dev": 0.11788546679722656, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.05024318055555556, "avg_duration_per_task_ms": 408710.0763888889, "avg_output_tokens_per_task": 26837.208333333332, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.543478, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.053712021739130436, "avg_duration_per_task_ms": 3699508.3260869565, "avg_output_tokens_per_task": 39104.608695652176, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.5390477142857143, "accuracy_std_dev": 0.06584534894240396, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03078722857142857, "avg_duration_per_task_ms": 382475.1971428571, "avg_output_tokens_per_task": 16102.351428571428, "last_run_timestamp": "2026-10-24T19:54:36.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.5384294010025062, "accuracy_std_dev": 0.2628453105716364, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.014483468671679198, "avg_duration_per_task_ms": 268751.037593985, "avg_output_tokens_per_task": 7227.022556390977, "last_run_timestamp": "2026-10-17T11:22:12.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5370371414141414, "accuracy_std_dev": 0.020577999999999985, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.02047550505050505, "avg_duration_per_task_ms": 144559.1616161616, "avg_output_tokens_per_task": 38541.121212121216, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5362941747851002, "accuracy_std_dev": 0.05033110710574323, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.010147799426934096, "avg_duration_per_task_ms": 180290.85100286533, "avg_output_tokens_per_task": 16030.240687679083, "last_run_timestamp": "2026-10-31T19:08:29.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5346869695431471, "accuracy_std_dev": 0.0181709902798389, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.01636143654822335, "avg_duration_per_task_ms": 1046919.6649746193, "avg_output_tokens_per_task": 68140.03553299492, "last_run_timestamp": "2026-10-31T19:53:39.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.534014, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.01161230612244898, "avg_duration_per_task_ms": 94666.5306122449, "avg_output_tokens_per_task": 16582.326530612245, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5333335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10219375, "avg_duration_per_task_ms": 375443.12, "avg_output_tokens_per_task": 17495.06, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5300001249999999, "accuracy_std_dev": 0.02027592305196919, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.066882445, "avg_duration_per_task_ms": 284395.4725, "avg_output_tokens_per_task": 4047.5225, "last_run_timestamp": "2026-10-31T19:28:47.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5285472267002518, "accuracy_std_dev": 0.03054949850959095, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.0026572493702770776, "avg_duration_per_task_ms": 371704.879093199, "avg_output_tokens_per_task": 6936.692695214106, "last_run_timestamp": "2026-10-31T19:13:20.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.522831205479452, "accuracy_std_dev": 0.027543964303555765, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.11947870547945207, "avg_duration_per_task_ms": 69921.19178082192, "avg_output_tokens_per_task": 2362.404109589041, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.5200001428571428, "accuracy_std_dev": 0.0256965935620634, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.028399222857142855, "avg_duration_per_task_ms": 421712.7342857143, "avg_output_tokens_per_task": 58758.36571428571, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5133333333333333, "accuracy_std_dev": 0.028803034820811636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12443134666666665, "avg_duration_per_task_ms": 52357.68, "avg_output_tokens_per_task": 2755.9866666666667, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.5016778724832214, "accuracy_std_dev": 0.03760315973167088, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.16471912416107382, "avg_duration_per_task_ms": 1996208.6208053692, "avg_output_tokens_per_task": 38529.68791946309, "last_run_timestamp": "2026-10-17T19:35:49.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.49888883333333334, "accuracy_std_dev": 0.028130992093043726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.026234426666666665, "avg_duration_per_task_ms": 626626.7566666667, "avg_output_tokens_per_task": 52920.51, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.4931975918367347, "accuracy_std_dev": 0.018578984814838502, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.022155443877551022, "avg_duration_per_task_ms": 1297132.918367347, "avg_output_tokens_per_task": 110956.9693877551, "last_run_timestamp": "2026-10-31T19:48:14.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48850570114942526, "accuracy_std_dev": 0.017439968302957548, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.031982068965517245, "avg_duration_per_task_ms": 642915.867816092, "avg_output_tokens_per_task": 59401.63505747126, "last_run_timestamp": "2026-10-23T09:26:25.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.48333349999999997, "accuracy_std_dev": 0.03666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04059527, "avg_duration_per_task_ms": 669013.45, "avg_output_tokens_per_task": 93900.79, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.48047614285714285, "accuracy_std_dev": 0.02256761169355368, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02984426, "avg_duration_per_task_ms": 354083.21428571426, "avg_output_tokens_per_task": 63173.58857142857, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.48, "accuracy_std_dev": 0.03496018370232054, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.052433775, "avg_duration_per_task_ms": 89289.75, "avg_output_tokens_per_task": 2759.585, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47983850806451617, "accuracy_std_dev": 0.010328084689815433, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.16880521774193546, "avg_duration_per_task_ms": 2361254.620967742, "avg_output_tokens_per_task": 44884.471774193546, "last_run_timestamp": "2026-10-17T19:38:30.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.040637479166666664, "avg_duration_per_task_ms": 59566.541666666664, "avg_output_tokens_per_task": 2608.1666666666665, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.4777779999999999, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06735880666666666, "avg_duration_per_task_ms": 286720.9, "avg_output_tokens_per_task": 4080.2733333333335, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.47666650000000005, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12559784000000002, "avg_duration_per_task_ms": 52928.96, "avg_output_tokens_per_task": 2668.55, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.47555566666666665, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023880926666666667, "avg_duration_per_task_ms": 868904.6933333334, "avg_output_tokens_per_task": 23095.28, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.4754382631578947, "accuracy_std_dev": 0.054999999999999966, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.01757382105263158, "avg_duration_per_task_ms": 167109.45263157896, "avg_output_tokens_per_task": 28512.663157894738, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.475, "accuracy_std_dev": 0.025980762113533153, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.034838315, "avg_duration_per_task_ms": 40251.235, "avg_output_tokens_per_task": 3653.015, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.471111, "accuracy_std_dev": 0.008314764498569181, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021167693333333334, "avg_duration_per_task_ms": 745067.6466666666, "avg_output_tokens_per_task": 19698.2, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.47101399999999993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.08111417391304347, "avg_duration_per_task_ms": 51984.04347826087, "avg_output_tokens_per_task": 3590.7391304347825, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.47, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03551402, "avg_duration_per_task_ms": 437816.58, "avg_output_tokens_per_task": 20922.04, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.46820784971098267, "accuracy_std_dev": 0.04097451831844482, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.027368598265895955, "avg_duration_per_task_ms": 2298741.832369942, "avg_output_tokens_per_task": 136175.1647398844, "last_run_timestamp": "2026-10-24T19:05:04.624Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.466667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03567932, "avg_duration_per_task_ms": 645558.66, "avg_output_tokens_per_task": 31749.44, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.46666650000000004, "accuracy_std_dev": 0.02108179836375446, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.023839585000000003, "avg_duration_per_task_ms": 125170.4, "avg_output_tokens_per_task": 3749.6, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.4655553333333334, "accuracy_std_dev": 0.012861993011280265, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020687186666666666, "avg_duration_per_task_ms": 683546.4733333334, "avg_output_tokens_per_task": 19239.053333333333, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4644446666666666, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04325750666666667, "avg_duration_per_task_ms": 738509.7133333334, "avg_output_tokens_per_task": 100007.46, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.46333349999999995, "accuracy_std_dev": 0.046308230610227374, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025355005, "avg_duration_per_task_ms": 58190.01, "avg_output_tokens_per_task": 3726.025, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.46166674999999996, "accuracy_std_dev": 0.04010416645047619, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04448468, "avg_duration_per_task_ms": 738816.635, "avg_output_tokens_per_task": 95390.315, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011152899999999999, "avg_duration_per_task_ms": 87357.46, "avg_output_tokens_per_task": 9087.9, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00632216, "avg_duration_per_task_ms": 38771.86, "avg_output_tokens_per_task": 1211.54, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.4597700689655172, "accuracy_std_dev": 0.17656167724922991, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07236286206896553, "avg_duration_per_task_ms": 580100.433908046, "avg_output_tokens_per_task": 10024.175287356322, "last_run_timestamp": "2026-10-31T19:59:00.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.45875, "accuracy_std_dev": 0.14443306119704724, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0279354525, "avg_duration_per_task_ms": 219700.04, "avg_output_tokens_per_task": 18638.5075, "last_run_timestamp": "2026-10-31T19:46:58.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.45833349999999995, "accuracy_std_dev": 0.036552937258857866, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.019015, "avg_duration_per_task_ms": 100068.645, "avg_output_tokens_per_task": 3610.935, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.453333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03731064, "avg_duration_per_task_ms": 268154, "avg_output_tokens_per_task": 3892.22, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.45114957241379305, "accuracy_std_dev": 0.023020145226518452, "run_count": 6, "total_questions": 290, "avg_cost_per_task_usd": 0.0516089, "avg_duration_per_task_ms": 110224.24482758621, "avg_output_tokens_per_task": 64863.19655172414, "last_run_timestamp": "2026-10-31T19:03:34.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.45111100000000004, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023389446666666668, "avg_duration_per_task_ms": 54689.44666666666, "avg_output_tokens_per_task": 2398.8933333333334, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.4497488994974874, "accuracy_std_dev": 0.02675352548730728, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.023178929648241205, "avg_duration_per_task_ms": 1975750.9296482413, "avg_output_tokens_per_task": 14402.733668341709, "last_run_timestamp": "2026-11-04T19:34:19.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4497487512562814, "accuracy_std_dev": 0.043480269304018805, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.015482552763819097, "avg_duration_per_task_ms": 129064.97989949748, "avg_output_tokens_per_task": 3318.743718592965, "last_run_timestamp": "2026-10-31T18:58:15.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.446667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.087889, "avg_duration_per_task_ms": 78557.44, "avg_output_tokens_per_task": 4155.32, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4466668, "accuracy_std_dev": 0.030768057029328312, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021730787999999997, "avg_duration_per_task_ms": 222860.848, "avg_output_tokens_per_task": 3530.168, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.44333300000000003, "accuracy_std_dev": 0.009999999999999981, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023192820000000003, "avg_duration_per_task_ms": 58714.73, "avg_output_tokens_per_task": 2330.08, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.442176887755102, "accuracy_std_dev": 0.05508412293199829, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.049728214285714285, "avg_duration_per_task_ms": 74838.58163265306, "avg_output_tokens_per_task": 3054.173469387755, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.4414284285714286, "accuracy_std_dev": 0.029160501195267273, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.039515631428571424, "avg_duration_per_task_ms": 280456.21142857143, "avg_output_tokens_per_task": 3528.7685714285712, "last_run_timestamp": "2026-10-31T19:56:10.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.44, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03130282, "avg_duration_per_task_ms": 680376.23, "avg_output_tokens_per_task": 16688.33, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4377776666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10492657333333334, "avg_duration_per_task_ms": 892034.1333333333, "avg_output_tokens_per_task": 22149.68666666667, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.4372221666666667, "accuracy_std_dev": 0.029465364687016663, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.053356796666666664, "avg_duration_per_task_ms": 150036.94, "avg_output_tokens_per_task": 3684.5133333333333, "last_run_timestamp": "2026-10-17T19:11:29.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.43666675, "accuracy_std_dev": 0.08698667522493028, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.22938595999999997, "avg_duration_per_task_ms": 120240.42, "avg_output_tokens_per_task": 3735.89, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.43378998630136983, "accuracy_std_dev": 0.04334328866413141, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.05113669178082191, "avg_duration_per_task_ms": 70393.37671232877, "avg_output_tokens_per_task": 2340.2671232876714, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.43333333333333335, "accuracy_std_dev": 0.02372680934208295, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07764511333333333, "avg_duration_per_task_ms": 794132.34, "avg_output_tokens_per_task": 20856.233333333334, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.4322221666666667, "accuracy_std_dev": 0.01651776073823432, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21270556333333332, "avg_duration_per_task_ms": 46305.06, "avg_output_tokens_per_task": 2266.36, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.43142857142857144, "accuracy_std_dev": 0.030750162879276607, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007827288571428571, "avg_duration_per_task_ms": 194648.64, "avg_output_tokens_per_task": 4575.702857142857, "last_run_timestamp": "2026-10-31T19:46:05.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.431111, "accuracy_std_dev": 0.00314238253559302, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.24174756666666666, "avg_duration_per_task_ms": 46576.24, "avg_output_tokens_per_task": 2473.9733333333334, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.43027193877551023, "accuracy_std_dev": 0.04094127400008455, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.06303705612244899, "avg_duration_per_task_ms": 258525.35714285713, "avg_output_tokens_per_task": 2431.734693877551, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.4301802162162162, "accuracy_std_dev": 0.02412263108922131, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06743897297297298, "avg_duration_per_task_ms": 360892.5135135135, "avg_output_tokens_per_task": 3318.9054054054054, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.42888899999999996, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05722675333333334, "avg_duration_per_task_ms": 196077.84666666668, "avg_output_tokens_per_task": 2331.213333333333, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.4283335, "accuracy_std_dev": 0.013016997436044923, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025223935000000003, "avg_duration_per_task_ms": 228221.25, "avg_output_tokens_per_task": 3524.31, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.42666699999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02109498, "avg_duration_per_task_ms": 190969.76, "avg_output_tokens_per_task": 3644.16, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.42666650000000006, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21223042999999997, "avg_duration_per_task_ms": 51685.1, "avg_output_tokens_per_task": 2189.48, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.42555566666666667, "accuracy_std_dev": 0.02910172028515764, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.027489039999999996, "avg_duration_per_task_ms": 68221.56666666667, "avg_output_tokens_per_task": 2795.5733333333333, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.4255553333333334, "accuracy_std_dev": 0.034462310778910675, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029985173333333334, "avg_duration_per_task_ms": 822675.42, "avg_output_tokens_per_task": 18074.12, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4253334, "accuracy_std_dev": 0.03222148627608603, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.052599823999999996, "avg_duration_per_task_ms": 127600.244, "avg_output_tokens_per_task": 3365.64, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.42499975000000006, "accuracy_std_dev": 0.011902485735656231, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03875372, "avg_duration_per_task_ms": 152646.77, "avg_output_tokens_per_task": 3373.325, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.4242807969543147, "accuracy_std_dev": 0.029334918915006395, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.005077210659898477, "avg_duration_per_task_ms": 361279.1116751269, "avg_output_tokens_per_task": 4832.657360406091, "last_run_timestamp": "2026-10-31T19:55:29.787Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.42333344444444443, "accuracy_std_dev": 0.030307058217558316, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024007242222222225, "avg_duration_per_task_ms": 421871.2622222222, "avg_output_tokens_per_task": 3556.5755555555556, "last_run_timestamp": "2026-10-31T19:22:37.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.42333325, "accuracy_std_dev": 0.05763875528832576, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.563229095, "avg_duration_per_task_ms": 116619.195, "avg_output_tokens_per_task": 3538.205, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Modal", "accuracy": 0.423333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00707156, "avg_duration_per_task_ms": 19040.86, "avg_output_tokens_per_task": 3450.68, "last_run_timestamp": "2026-11-04T09:13:34.415Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.42176859183673465, "accuracy_std_dev": 0.029027999999999998, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.033904112244897956, "avg_duration_per_task_ms": 521650.17346938775, "avg_output_tokens_per_task": 5726.459183673469, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41888899999999996, "accuracy_std_dev": 0.014614444065147779, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01463076, "avg_duration_per_task_ms": 76549.69333333333, "avg_output_tokens_per_task": 8814.213333333333, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.416667, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.19647352999999998, "avg_duration_per_task_ms": 53166.02, "avg_output_tokens_per_task": 2086.91, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.41555566666666666, "accuracy_std_dev": 0.015713798296048267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01727216, "avg_duration_per_task_ms": 121607.27333333333, "avg_output_tokens_per_task": 2838.9866666666667, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4149376141078838, "accuracy_std_dev": 0.02248141178485016, "run_count": 5, "total_questions": 241, "avg_cost_per_task_usd": 0.01079711203319502, "avg_duration_per_task_ms": 83125.57676348547, "avg_output_tokens_per_task": 3009.3568464730292, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.41363020242914983, "accuracy_std_dev": 0.021563261057641545, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.0543252995951417, "avg_duration_per_task_ms": 608248.056680162, "avg_output_tokens_per_task": 23841.582995951416, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4132554619883041, "accuracy_std_dev": 0.026064865319476507, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.06276042690058481, "avg_duration_per_task_ms": 235574.4707602339, "avg_output_tokens_per_task": 5011.222222222223, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.41, "accuracy_std_dev": 0.0317979384394649, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07720068, "avg_duration_per_task_ms": 347548.415, "avg_output_tokens_per_task": 16051.36, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.40833325000000004, "accuracy_std_dev": 0.01907573356879102, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.075146255, "avg_duration_per_task_ms": 435207.485, "avg_output_tokens_per_task": 11776.71, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.406667, "accuracy_std_dev": 0.0565685424949238, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.055826546666666664, "avg_duration_per_task_ms": 125468.13333333333, "avg_output_tokens_per_task": 3336.5933333333332, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.40499975000000005, "accuracy_std_dev": 0.028431130239360863, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07697062, "avg_duration_per_task_ms": 115787.03, "avg_output_tokens_per_task": 3079.1, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.40380291275167784, "accuracy_std_dev": 0.008840018715415086, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.07104265771812081, "avg_duration_per_task_ms": 235544.13422818793, "avg_output_tokens_per_task": 8668.979865771813, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40333325000000003, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.023807599999999998, "avg_duration_per_task_ms": 77145.375, "avg_output_tokens_per_task": 2775.135, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.401667, "accuracy_std_dev": 0.025000000000000022, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01796061, "avg_duration_per_task_ms": 1329879.6, "avg_output_tokens_per_task": 125682.59, "last_run_timestamp": "2026-10-31T20:05:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.40166650000000004, "accuracy_std_dev": 0.00866035026716588, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.034973950000000004, "avg_duration_per_task_ms": 178771.545, "avg_output_tokens_per_task": 3509.995, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.40000025, "accuracy_std_dev": 0.018257509870940786, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06178068, "avg_duration_per_task_ms": 95175.335, "avg_output_tokens_per_task": 2906.275, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.39926732967032963, "accuracy_std_dev": 0.017592999999999998, "run_count": 2, "total_questions": 91, "avg_cost_per_task_usd": 0.009130032967032966, "avg_duration_per_task_ms": 125085.65934065935, "avg_output_tokens_per_task": 2027.8351648351647, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.39833349999999995, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06903712000000001, "avg_duration_per_task_ms": 315593.59, "avg_output_tokens_per_task": 7308.28, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.39666674999999996, "accuracy_std_dev": 0.02768865592056609, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027719889999999997, "avg_duration_per_task_ms": 277229.35, "avg_output_tokens_per_task": 4205.25, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.3933333333333333, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015131493333333334, "avg_duration_per_task_ms": 100578.96, "avg_output_tokens_per_task": 613.0666666666667, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.39166674999999995, "accuracy_std_dev": 0.127355609962567, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.020578967500000003, "avg_duration_per_task_ms": 226485.375, "avg_output_tokens_per_task": 3325.03, "last_run_timestamp": "2026-10-31T19:18:14.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3866665, "accuracy_std_dev": 0.00816496581438035, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025150575, "avg_duration_per_task_ms": 193350.84, "avg_output_tokens_per_task": 4372.65, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.38279137398373986, "accuracy_std_dev": 0.019122202306219867, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.05721597154471545, "avg_duration_per_task_ms": 352471.3292682927, "avg_output_tokens_per_task": 3819.8211382113823, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.38, "accuracy_std_dev": 0.005443582705045132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028440120000000006, "avg_duration_per_task_ms": 82079.82666666666, "avg_output_tokens_per_task": 4234.44, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.38, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.020695199999999997, "avg_duration_per_task_ms": 192718, "avg_output_tokens_per_task": 3853.66, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.3738005794743429, "accuracy_std_dev": 0.022831389807948622, "run_count": 16, "total_questions": 799, "avg_cost_per_task_usd": 0.013106496871088861, "avg_duration_per_task_ms": 40746.92365456821, "avg_output_tokens_per_task": 767.6082603254067, "last_run_timestamp": "2026-10-31T19:10:53.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.37, "accuracy_std_dev": 0.0481895677973978, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.05701098999999999, "avg_duration_per_task_ms": 93841.05, "avg_output_tokens_per_task": 3849.31, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.3694728520408163, "accuracy_std_dev": 0.1273174900715113, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.05665731377551021, "avg_duration_per_task_ms": 831256.9948979592, "avg_output_tokens_per_task": 9740.272959183674, "last_run_timestamp": "2026-10-31T19:55:08.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.36799980000000004, "accuracy_std_dev": 0.019504976005112133, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.032627564, "avg_duration_per_task_ms": 131122.06, "avg_output_tokens_per_task": 4369.504, "last_run_timestamp": "2026-10-23T08:14:36.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.36666650000000006, "accuracy_std_dev": 0.02943920288917484, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.05628207, "avg_duration_per_task_ms": 69776.165, "avg_output_tokens_per_task": 4432.06, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35678391959798994, "accuracy_std_dev": 0.031780867542438175, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.07306190452261307, "avg_duration_per_task_ms": 404962.46482412063, "avg_output_tokens_per_task": 8082.469849246231, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.3534484195402299, "accuracy_std_dev": 0.02330663243957542, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07739280172413791, "avg_duration_per_task_ms": 206617.80747126436, "avg_output_tokens_per_task": 7421.793103448276, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.35333325000000004, "accuracy_std_dev": 0.034480082818744796, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09413336750000001, "avg_duration_per_task_ms": 27568.8675, "avg_output_tokens_per_task": 1082.295, "last_run_timestamp": "2026-10-31T18:48:48.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.34013619387755106, "accuracy_std_dev": 0.08215978685882466, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.5213178979591837, "avg_duration_per_task_ms": 540508.3367346938, "avg_output_tokens_per_task": 21094.061224489797, "last_run_timestamp": "2026-10-10T18:31:45.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.3345927808564232, "accuracy_std_dev": 0.13502996743634121, "run_count": 16, "total_questions": 794, "avg_cost_per_task_usd": 0.03743722166246851, "avg_duration_per_task_ms": 517162.338790932, "avg_output_tokens_per_task": 129436.19017632242, "last_run_timestamp": "2026-10-31T20:07:06.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.32996641919191916, "accuracy_std_dev": 0.04648910770394905, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.05162643181818182, "avg_duration_per_task_ms": 79675.6186868687, "avg_output_tokens_per_task": 1323.4722222222222, "last_run_timestamp": "2026-10-31T19:07:35.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.32666671428571425, "accuracy_std_dev": 0.02666675000131537, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07288134285714286, "avg_duration_per_task_ms": 131296.76, "avg_output_tokens_per_task": 7380.06, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.3249161313131313, "accuracy_std_dev": 0.3215350090739537, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.004530868686868687, "avg_duration_per_task_ms": 258173.5101010101, "avg_output_tokens_per_task": 8291.585858585859, "last_run_timestamp": "2026-10-24T19:32:27.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3150683150684931, "accuracy_std_dev": 0.04608944973514969, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.01928564383561644, "avg_duration_per_task_ms": 173538.2602739726, "avg_output_tokens_per_task": 1715.5753424657535, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.30666649999999995, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01779966, "avg_duration_per_task_ms": 55498.41, "avg_output_tokens_per_task": 3406.81, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.29, "accuracy_std_dev": 0.02821877494387971, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17210771, "avg_duration_per_task_ms": 134359.03333333333, "avg_output_tokens_per_task": 1879.0433333333333, "last_run_timestamp": "2026-10-17T19:19:08.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.2844446666666667, "accuracy_std_dev": 0.02997950740459592, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025635820000000004, "avg_duration_per_task_ms": 67806.75333333333, "avg_output_tokens_per_task": 4107.166666666667, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07945186, "avg_duration_per_task_ms": 632916.72, "avg_output_tokens_per_task": 5086.04, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.25166675, "accuracy_std_dev": 0.06998009439253064, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.014117339999999999, "avg_duration_per_task_ms": 22717.98, "avg_output_tokens_per_task": 1149.03, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.24222200000000002, "accuracy_std_dev": 0.10100740224689805, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14696888000000002, "avg_duration_per_task_ms": 457890.76, "avg_output_tokens_per_task": 10647.186666666666, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07327475, "avg_duration_per_task_ms": 662655.63, "avg_output_tokens_per_task": 5322.46, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.23504264102564099, "accuracy_std_dev": 0.060676152257291324, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.004586492307692308, "avg_duration_per_task_ms": 16018.45641025641, "avg_output_tokens_per_task": 1693.246153846154, "last_run_timestamp": "2026-10-24T19:30:38.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.22578841216216217, "accuracy_std_dev": 0.3171451865228027, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.007187790540540541, "avg_duration_per_task_ms": 27424.14864864865, "avg_output_tokens_per_task": 7417.02027027027, "last_run_timestamp": "2026-10-31T19:36:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.19574940268456376, "accuracy_std_dev": 0.18279813689209806, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.2264336979865772, "avg_duration_per_task_ms": 1382638.3389261744, "avg_output_tokens_per_task": 8657.936241610738, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.17942213775510205, "accuracy_std_dev": 0.009739601557558703, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.010118459183673469, "avg_duration_per_task_ms": 155344.85714285713, "avg_output_tokens_per_task": 4996.790816326531, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.17185177777777777, "accuracy_std_dev": 0.22268352465714294, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05974023333333334, "avg_duration_per_task_ms": 93439.47333333333, "avg_output_tokens_per_task": 15042.673333333334, "last_run_timestamp": "2026-10-31T19:46:05.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.162222, "accuracy_std_dev": 0.10740478872936718, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010201566666666667, "avg_duration_per_task_ms": 48420.386666666665, "avg_output_tokens_per_task": 838.2466666666667, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.15221598843930634, "accuracy_std_dev": 0.22421725648430124, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.006188393063583815, "avg_duration_per_task_ms": 494644.65606936417, "avg_output_tokens_per_task": 41928.95953757226, "last_run_timestamp": "2026-10-31T19:13:59.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.14, "accuracy_std_dev": 0.014142135623730954, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0072234, "avg_duration_per_task_ms": 76455.195, "avg_output_tokens_per_task": 2322.375, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.111111, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.007342933333333333, "avg_duration_per_task_ms": 57269.24, "avg_output_tokens_per_task": 2382.0866666666666, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.11, "accuracy_std_dev": 0.009999999999999995, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.006762440000000001, "avg_duration_per_task_ms": 64306.02, "avg_output_tokens_per_task": 2312.51, "last_run_timestamp": "2026-10-01T19:04:36.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.10666677777777778, "accuracy_std_dev": 0.18531230895963338, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0021768155555555555, "avg_duration_per_task_ms": 5484.648888888889, "avg_output_tokens_per_task": 710.5644444444445, "last_run_timestamp": "2026-10-17T19:48:24.252Z" } ] }